mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-09-12 16:09:01 +02:00
Rohanjames1997
982937a333
* Test for nrc=2 as well | i8mm kernels * Trigger only on supported HW * Remove trailing whitespace * Address review comment * test: properly prepare nrc=2 inputs with independent data per row * tests : make nrc=2 dot product inputs distinct Assisted-by: Kiro * tests : use non-trivial strides in nrc=2 dot product test * tests : fail nrc=2 dot product test on non-finite errors
274 lines
11 KiB
C++
274 lines
11 KiB
C++
// Unit tests for quantization specific functions - quantize, dequantize and dot product
|
|
|
|
#include "ggml.h"
|
|
#include "ggml-cpu.h"
|
|
|
|
#undef NDEBUG
|
|
#include <assert.h>
|
|
#include <algorithm>
|
|
#include <cmath>
|
|
#include <math.h>
|
|
#include <stdio.h>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#if defined(_MSC_VER)
|
|
#pragma warning(disable: 4244 4267) // possible loss of data
|
|
#endif
|
|
|
|
constexpr float MAX_QUANTIZATION_REFERENCE_ERROR = 0.0001f;
|
|
constexpr float MAX_QUANTIZATION_TOTAL_ERROR = 0.002f;
|
|
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_BINARY = 0.025f;
|
|
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_TERNARY = 0.01f;
|
|
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_2BITS = 0.0075f;
|
|
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_3BITS = 0.0040f;
|
|
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS = 0.0050f;
|
|
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_FP4 = 0.0030f;
|
|
constexpr float MAX_DOT_PRODUCT_ERROR = 0.02f;
|
|
constexpr float MAX_DOT_PRODUCT_ERROR_LOWBIT = 0.04f;
|
|
constexpr float MAX_DOT_PRODUCT_ERROR_FP4 = 0.03f;
|
|
constexpr float MAX_DOT_PRODUCT_ERROR_BINARY = 0.40f;
|
|
constexpr float MAX_DOT_PRODUCT_ERROR_TERNARY = 0.15f;
|
|
|
|
static const char* RESULT_STR[] = {"ok", "FAILED"};
|
|
|
|
|
|
// Generate synthetic data
|
|
static void generate_data(float offset, size_t n, float * dst, float amplitude = 2.0f) {
|
|
for (size_t i = 0; i < n; i++) {
|
|
dst[i] = 0.1 + amplitude*cosf(i + offset);
|
|
}
|
|
}
|
|
|
|
// Calculate RMSE between two float arrays
|
|
static float array_rmse(const float * a1, const float * a2, size_t n) {
|
|
double sum = 0;
|
|
for (size_t i = 0; i < n; i++) {
|
|
double diff = a1[i] - a2[i];
|
|
sum += diff * diff;
|
|
}
|
|
return sqrtf(sum) / n;
|
|
}
|
|
|
|
// Total quantization error on test data
|
|
static float total_quantization_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data) {
|
|
std::vector<uint8_t> tmp_q(2*test_size);
|
|
std::vector<float> tmp_out(test_size);
|
|
|
|
qfns_cpu->from_float(test_data, tmp_q.data(), test_size);
|
|
qfns->to_float(tmp_q.data(), tmp_out.data(), test_size);
|
|
return array_rmse(test_data, tmp_out.data(), test_size);
|
|
}
|
|
|
|
// Total quantization error on test data
|
|
static float reference_quantization_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data) {
|
|
std::vector<uint8_t> tmp_q(2*test_size);
|
|
std::vector<float> tmp_out(test_size);
|
|
std::vector<float> tmp_out_ref(test_size);
|
|
|
|
// FIXME: why is done twice?
|
|
qfns_cpu->from_float(test_data, tmp_q.data(), test_size);
|
|
qfns->to_float(tmp_q.data(), tmp_out.data(), test_size);
|
|
|
|
qfns->from_float_ref(test_data, tmp_q.data(), test_size);
|
|
qfns->to_float(tmp_q.data(), tmp_out_ref.data(), test_size);
|
|
|
|
return array_rmse(tmp_out.data(), tmp_out_ref.data(), test_size);
|
|
}
|
|
|
|
static float dot_product(const float * a1, const float * a2, size_t test_size) {
|
|
double sum = 0;
|
|
for (size_t i = 0; i < test_size; i++) {
|
|
sum += a1[i] * a2[i];
|
|
}
|
|
return sum;
|
|
}
|
|
|
|
// Total dot product error
|
|
static float dot_product_error(const ggml_type_traits_cpu * qfns_cpu, ggml_type src0_type, size_t test_size,
|
|
const float * test_data1, const float * test_data2,
|
|
const float * test_data3, const float * test_data4,
|
|
const int nrc) {
|
|
const auto * vdot = ggml_get_type_traits_cpu(qfns_cpu->vec_dot_type);
|
|
const size_t pad = 64;
|
|
const size_t bx = ggml_row_size(src0_type, test_size) + pad;
|
|
const size_t by = ggml_row_size(qfns_cpu->vec_dot_type, test_size) + pad;
|
|
|
|
std::vector<uint8_t> tmp_q1(bx * nrc);
|
|
std::vector<uint8_t> tmp_q2(by * nrc);
|
|
|
|
qfns_cpu->from_float(test_data1, tmp_q1.data(), test_size);
|
|
vdot->from_float(test_data2, tmp_q2.data(), test_size);
|
|
|
|
if (nrc == 1) {
|
|
float result = INFINITY;
|
|
qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1);
|
|
|
|
const float dot_ref = dot_product(test_data1, test_data2, test_size);
|
|
return fabsf(result - dot_ref) / test_size;
|
|
}
|
|
|
|
// nrc == 2: kernel computes a 2x2 dot product matrix
|
|
// Output layout: s[0]=dot(vx0,vy0), s[1]=dot(vx1,vy0), s[bs]=dot(vx0,vy1), s[bs+1]=dot(vx1,vy1)
|
|
// row and output strides are padded, same as in the mul_mat path
|
|
qfns_cpu->from_float(test_data3, tmp_q1.data() + bx, test_size);
|
|
vdot->from_float(test_data4, tmp_q2.data() + by, test_size);
|
|
|
|
const size_t bs = 16;
|
|
std::vector<float> result(bs + 2, INFINITY);
|
|
qfns_cpu->vec_dot(test_size, result.data(), bs, tmp_q1.data(), bx, tmp_q2.data(), by, 2);
|
|
|
|
const float ref00 = dot_product(test_data1, test_data2, test_size);
|
|
const float ref10 = dot_product(test_data3, test_data2, test_size);
|
|
const float ref01 = dot_product(test_data1, test_data4, test_size);
|
|
const float ref11 = dot_product(test_data3, test_data4, test_size);
|
|
|
|
const auto err = [test_size](float val, float ref) {
|
|
const float e = fabsf(val - ref) / test_size;
|
|
return std::isfinite(e) ? e : INFINITY;
|
|
};
|
|
|
|
return std::max({err(result[0], ref00), err(result[1], ref10), err(result[bs], ref01), err(result[bs + 1], ref11)});
|
|
}
|
|
|
|
static int test_vec_dot_f32(bool verbose) {
|
|
const auto * f32 = ggml_get_type_traits_cpu(GGML_TYPE_F32);
|
|
int num_failed = 0;
|
|
for (int n : {1, 2, 3, 5, 7, 8, 15, 16, 17, 31, 33, 63, 67, 127, 129, 193, 255, 1023}) {
|
|
std::vector<float> a(n);
|
|
std::vector<float> b(n);
|
|
generate_data(0.0, n, a.data());
|
|
generate_data(1.0, n, b.data());
|
|
|
|
float result = 0.0f;
|
|
f32->vec_dot(n, &result, 0, a.data(), 0, b.data(), 0, 1);
|
|
const float ref = dot_product(a.data(), b.data(), n);
|
|
const float error = fabsf(result - ref) / n;
|
|
|
|
const bool failed = !(error < MAX_QUANTIZATION_REFERENCE_ERROR);
|
|
num_failed += failed;
|
|
if (failed || verbose) {
|
|
printf(" f32 vec_dot n=%4d: %s (ref=%f got=%f err=%f)\n",
|
|
n, RESULT_STR[failed], ref, result, error);
|
|
}
|
|
}
|
|
return num_failed;
|
|
}
|
|
|
|
static int test_vec_dot_q(bool verbose) {
|
|
int num_failed = 0;
|
|
|
|
const size_t test_size = 32 * 128;
|
|
|
|
std::vector<float> test_data(test_size);
|
|
std::vector<float> test_data2(test_size);
|
|
std::vector<float> test_data3(test_size);
|
|
std::vector<float> test_data4(test_size);
|
|
|
|
generate_data(0.0, test_data.size(), test_data.data());
|
|
generate_data(1.0, test_data2.size(), test_data2.data());
|
|
generate_data(3.0, test_data3.size(), test_data3.data(), 1.0f);
|
|
generate_data(4.0, test_data4.size(), test_data4.data(), 1.5f);
|
|
|
|
for (int i = 0; i < GGML_TYPE_COUNT; i++) {
|
|
ggml_type type = (ggml_type) i;
|
|
const auto * qfns = ggml_get_type_traits(type);
|
|
const auto * qfns_cpu = ggml_get_type_traits_cpu(type);
|
|
|
|
// deprecated - skip
|
|
if (qfns->blck_size == 0) {
|
|
continue;
|
|
}
|
|
|
|
const ggml_type ei = (ggml_type)i;
|
|
|
|
printf("Testing %s\n", ggml_type_name((ggml_type) i));
|
|
ggml_quantize_init(ei);
|
|
|
|
if (qfns_cpu->from_float && qfns->to_float) {
|
|
const float total_error = total_quantization_error(qfns, qfns_cpu, test_size, test_data.data());
|
|
const float max_quantization_error =
|
|
type == GGML_TYPE_Q1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_BINARY :
|
|
type == GGML_TYPE_TQ1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :
|
|
type == GGML_TYPE_TQ2_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :
|
|
type == GGML_TYPE_Q2_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :
|
|
type == GGML_TYPE_Q2_K ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS :
|
|
type == GGML_TYPE_IQ2_S ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS :
|
|
type == GGML_TYPE_Q3_K ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS :
|
|
type == GGML_TYPE_IQ3_S ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS :
|
|
type == GGML_TYPE_IQ3_XXS ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS :
|
|
type == GGML_TYPE_NVFP4 ? MAX_QUANTIZATION_TOTAL_ERROR_FP4 : MAX_QUANTIZATION_TOTAL_ERROR;
|
|
bool failed = !(total_error < max_quantization_error);
|
|
num_failed += failed;
|
|
if (failed || verbose) {
|
|
printf("%5s absolute quantization error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], total_error);
|
|
}
|
|
|
|
const float reference_error = reference_quantization_error(qfns, qfns_cpu, test_size, test_data.data());
|
|
failed = !(reference_error < MAX_QUANTIZATION_REFERENCE_ERROR);
|
|
num_failed += failed;
|
|
if (failed || verbose) {
|
|
printf("%5s reference implementation error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], reference_error);
|
|
}
|
|
|
|
const float vec_dot_error = dot_product_error(qfns_cpu, type, test_size, test_data.data(), test_data2.data(), nullptr, nullptr, 1);
|
|
const float max_allowed_error = type == GGML_TYPE_Q2_K || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ2_XXS ||
|
|
type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ2_S
|
|
? MAX_DOT_PRODUCT_ERROR_LOWBIT
|
|
: type == GGML_TYPE_Q1_0
|
|
? MAX_DOT_PRODUCT_ERROR_BINARY
|
|
: type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0 || type == GGML_TYPE_Q2_0
|
|
? MAX_DOT_PRODUCT_ERROR_TERNARY
|
|
: type == GGML_TYPE_NVFP4
|
|
? MAX_DOT_PRODUCT_ERROR_FP4
|
|
: MAX_DOT_PRODUCT_ERROR;
|
|
failed = !(vec_dot_error < max_allowed_error);
|
|
num_failed += failed;
|
|
if (failed || verbose) {
|
|
printf("%5s dot product error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error);
|
|
}
|
|
|
|
// Test nrc=2 path for types that support it
|
|
if (qfns_cpu->nrows == 2) {
|
|
const float vec_dot_error_nrc2 = dot_product_error(qfns_cpu, type, test_size, test_data.data(), test_data2.data(), test_data3.data(), test_data4.data(), 2);
|
|
failed = !(vec_dot_error_nrc2 < max_allowed_error);
|
|
num_failed += failed;
|
|
if (failed || verbose) {
|
|
printf("%5s dot product error (nrc=2): %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error_nrc2);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return num_failed;
|
|
}
|
|
|
|
int main(int argc, char * argv[]) {
|
|
bool verbose = false;
|
|
|
|
std::string arg;
|
|
for (int i = 1; i < argc; i++) {
|
|
arg = argv[i];
|
|
|
|
if (arg == "-v") {
|
|
verbose = true;
|
|
} else {
|
|
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
ggml_cpu_init();
|
|
|
|
int num_failed = 0;
|
|
|
|
num_failed += test_vec_dot_f32(verbose);
|
|
num_failed += test_vec_dot_q(verbose);
|
|
|
|
if (num_failed || verbose) {
|
|
printf("%d tests failed\n", num_failed);
|
|
}
|
|
|
|
return num_failed > 0;
|
|
}
|