a better default rms_norm_eps

2026-09-19 09:15:18 +02:00 · 2023-07-25 22:51:01 +08:00
parent 3e68cdd26a
commit 0c26799e77
1 changed files with 1 additions and 1 deletions
@@ -1030,7 +1030,7 @@ static void llama_model_load_internal(
    auto & hparams = model.hparams;

    // TODO: read from file
-    hparams.f_rms_norm_eps = rms_norm_eps;
+    hparams.f_rms_norm_eps = 5e-6;

    {
        switch (hparams.n_layer) {