diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 08eb042352..5b81554dea 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -975,6 +975,9 @@ private:
             }
         }
 
+        llama_backend_init();
+        llama_numa_init(params.numa);
+
         load_progress_data load_progress_text  (this, "text_model");
         load_progress_data load_progress_mmproj(this, "mmproj_model");
         load_progress_data load_progress_spec  (this, "spec_model");
diff --git a/tools/server/server.cpp b/tools/server/server.cpp
index 9ff5faefd9..e8030a5c20 100644
--- a/tools/server/server.cpp
+++ b/tools/server/server.cpp
@@ -106,9 +106,6 @@ int llama_server(int argc, char ** argv) {
         return 1;
     }
 
-    llama_backend_init();
-    llama_numa_init(params.numa);
-
     return llama_server(params, argc, argv);
 }