Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 17:24:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

WIP OCP FP8 E4M3 support

Browse Source
This commit is contained in:
Oliver Simons
2026-08-21 18:38:22 +02:00
parent 8b4b3558f1
commit dd53768baf
19 changed files with 353 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-backend-ops.cpp
+12 -4
View File
@@ -84,7 +84,7 @@ static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float m
if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_I32) {
ggml_backend_tensor_set(tensor, data.data(), 0, nels * sizeof(float));
} else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) {
} else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F8_E4M3 || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) {
GGML_ASSERT(nels % ggml_blck_size(tensor->type) == 0);
// dummy importance matrix
@@ -258,7 +258,7 @@ static std::vector<float> tensor_to_float(const ggml_tensor * t) {
const auto * tt = ggml_get_type_traits(t->type);
size_t bs = ggml_blck_size(t->type);
std::vector<float> vq(ggml_blck_size(t->type));
bool quantized = ggml_is_quantized(t->type);
bool uses_type_conversion = ggml_is_quantized(t->type) || t->type == GGML_TYPE_F8_E4M3;
// access elements by index to avoid gaps in views
for (int64_t i3 = 0; i3 < t->ne[3]; i3++) {
@@ -280,7 +280,7 @@ static std::vector<float> tensor_to_float(const ggml_tensor * t) {
tv.push_back((float)*(int16_t *) &buf[i]);
} else if (t->type == GGML_TYPE_I8) {
tv.push_back((float)*(int8_t *) &buf[i]);
} else if (quantized) {
} else if (uses_type_conversion) {
tt->to_float(&buf[i], vq.data(), bs);
tv.insert(tv.end(), vq.begin(), vq.end());
} else {
@@ -4639,6 +4639,9 @@ struct test_mul_mat : public test_case {
if ((type_a == GGML_TYPE_MXFP4 || type_a == GGML_TYPE_NVFP4) && backend_has_feature(backend, "BLACKWELL_NATIVE_FP4")) {
return 2e-2;
}
if (type_a == GGML_TYPE_F8_E4M3 && backend_has_feature(backend, "NATIVE_FP8")) {
return 5e-3;
}
return max_nmse_err();
}
@@ -4840,6 +4843,9 @@ struct test_mul_mat_id : public test_case {
if ((type_a == GGML_TYPE_MXFP4 || type_a == GGML_TYPE_NVFP4) && backend_has_feature(backend, "BLACKWELL_NATIVE_FP4")) {
return 2e-2;
}
if (type_a == GGML_TYPE_F8_E4M3 && backend_has_feature(backend, "NATIVE_FP8")) {
return 5e-3;
}
return max_nmse_err();
}
@@ -8559,6 +8565,7 @@ static const ggml_type all_types[] = {
GGML_TYPE_Q1_0,
GGML_TYPE_Q2_0,
GGML_TYPE_MXFP4, GGML_TYPE_NVFP4,
GGML_TYPE_F8_E4M3,
GGML_TYPE_Q2_K, GGML_TYPE_Q3_K,
GGML_TYPE_Q4_K, GGML_TYPE_Q5_K,
GGML_TYPE_Q6_K,
@@ -8578,6 +8585,7 @@ static const ggml_type base_types[] = {
GGML_TYPE_Q4_1, // for I8MM tests
GGML_TYPE_Q4_K,
GGML_TYPE_MXFP4, GGML_TYPE_NVFP4, // TODO: or "other"
GGML_TYPE_F8_E4M3,
GGML_TYPE_IQ2_XXS
};
@@ -10434,7 +10442,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
continue;
}
for (bool with_lane_scale : {false, true}) {
if (with_lane_scale && type != GGML_TYPE_NVFP4) {
if (with_lane_scale && type != GGML_TYPE_NVFP4 && type != GGML_TYPE_F8_E4M3) {
continue;
}
test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256,