diff --git a/tools/cli/CMakeLists.txt b/tools/cli/CMakeLists.txt index 2fa648bf0c..8449cdbaff 100644 --- a/tools/cli/CMakeLists.txt +++ b/tools/cli/CMakeLists.txt @@ -8,7 +8,7 @@ add_library(${TARGET} cli.cpp set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ../server) -target_link_libraries(${TARGET} PUBLIC server-context llama-common ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(${TARGET} PUBLIC llama-server-impl llama-common ${CMAKE_THREAD_LIBS_INIT}) if(LLAMA_TOOLS_INSTALL) install(TARGETS ${TARGET} LIBRARY) diff --git a/tools/cli/cli-context.cpp b/tools/cli/cli-context.cpp index f5af5ef3b8..dfc2c9f6b9 100644 --- a/tools/cli/cli-context.cpp +++ b/tools/cli/cli-context.cpp @@ -41,56 +41,6 @@ static int arg_num_values(const common_arg & opt) { return 0; } -// keep only the args that llama-server understands, so that the remainder -// of the command line can be forwarded to the spawned server child -static std::vector filter_server_args(int argc, char ** argv) { - std::map cli_n_values; // arg -> number of values - std::set server_args; - - common_params dummy_cli; - auto ctx_cli = common_params_parser_init(dummy_cli, LLAMA_EXAMPLE_CLI); - for (const auto & opt : ctx_cli.options) { - for (const char * a : opt.args) { - cli_n_values[a] = arg_num_values(opt); - } - for (const char * a : opt.args_neg) { - cli_n_values[a] = 0; - } - } - - common_params dummy_server; - auto ctx_server = common_params_parser_init(dummy_server, LLAMA_EXAMPLE_SERVER); - for (const auto & opt : ctx_server.options) { - for (const char * a : opt.args) { - server_args.insert(a); - } - for (const char * a : opt.args_neg) { - server_args.insert(a); - } - } - - std::vector result; - for (int i = 1; i < argc; i++) { - const std::string arg = argv[i]; - auto it = cli_n_values.find(arg); - if (it == cli_n_values.end()) { - // not a known arg (should not happen when parsing succeeded) - continue; - } - const bool forward = server_args.count(arg) > 0; - if (forward) { - result.push_back(arg); - } - for (int j = 0; j < it->second && i + 1 < argc; j++) { - i++; - if (forward) { - result.push_back(argv[i]); - } - } - } - return result; -} - static std::string format_error_message(const json & err) { if (err.contains("error") && err.at("error").is_object()) { const auto & e = err.at("error"); @@ -113,7 +63,7 @@ static std::string media_type_from_ext(const std::string & fname) { return "image"; } -bool cli_context::init(int argc, char ** argv) { +bool cli_context::init() { std::optional spinner; if (!params.server_base.empty()) { @@ -138,7 +88,7 @@ bool cli_context::init(int argc, char ** argv) { spinner.emplace("Loading model..."); server.emplace(); - if (!server->start(filter_server_args(argc, argv))) { + if (!server->start(params)) { view::show_error("server start failed"); return false; } diff --git a/tools/cli/cli-context.h b/tools/cli/cli-context.h index ef55b2c9b4..cbf6729d6d 100644 --- a/tools/cli/cli-context.h +++ b/tools/cli/cli-context.h @@ -62,7 +62,7 @@ struct cli_context { // connect to --server-base or spawn a local llama-server child; // argc/argv are needed to forward the server-relevant args to the child - bool init(int argc, char ** argv); + bool init(); // run the interactive chat loop, returns the process exit code int run(); diff --git a/tools/cli/cli-server.h b/tools/cli/cli-server.h index 8e6e388ce3..41f860af86 100644 --- a/tools/cli/cli-server.h +++ b/tools/cli/cli-server.h @@ -8,7 +8,7 @@ // note: in the future, we may have a server running as daemon and the CLI can connect to it automatically // llama_server will be available as a dynamic library symbol -int llama_server(int argc, char ** argv); +int llama_server(common_params & params, int argc, char ** argv); struct cli_server { std::thread th; @@ -24,26 +24,16 @@ struct cli_server { } } - bool start(std::vector args) { + bool start(common_params & params) { port = common_http_get_free_port(); if (port <= 0) { fprintf(stderr, "failed to get a free port\n"); exit(1); } - th = std::thread([&, args_ = args]() { - auto args = args_; // copy to modify - args.push_back("--port"); - args.push_back(std::to_string(port)); - - // convert to char* array - std::vector argv; - for (auto & arg : args) { - argv.push_back(arg.data()); - } - argv.push_back(nullptr); - - int res = llama_server(static_cast(args.size()), argv.data()); + th = std::thread([&]() { + // argc / argv are only used in router mode, we can skip them for now + int res = llama_server(params, 0, nullptr); if (res != 0) { fprintf(stderr, "llama_server exited with code %d\n", res); } diff --git a/tools/cli/cli.cpp b/tools/cli/cli.cpp index f16f18294e..2778b98c0b 100644 --- a/tools/cli/cli.cpp +++ b/tools/cli/cli.cpp @@ -182,7 +182,7 @@ int llama_cli(int argc, char ** argv) { cli_context ctx_cli(params); - if (!ctx_cli.init(argc, argv)) { + if (!ctx_cli.init()) { ctx_cli.shutdown(); return 1; } diff --git a/tools/server/server.cpp b/tools/server/server.cpp index dd4b1c507c..b5902458c8 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -21,6 +21,12 @@ #include #endif +// satisfies -Wmissing-declarations (used by llama command) +int llama_server(int argc, char ** argv); + +// to be used via CLI (argc / argv are used by router mode only) +int llama_server(common_params & params, int argc, char ** argv); + static std::function shutdown_handler; static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT; @@ -71,9 +77,6 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t }; } -// satisfies -Wmissing-declarations -int llama_server(int argc, char ** argv); - int llama_server(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); @@ -89,6 +92,10 @@ int llama_server(int argc, char ** argv) { llama_backend_init(); llama_numa_init(params.numa); + return llama_server(params, argc, argv); +} + +int llama_server(common_params & params, int argc, char ** argv) { // router server never loads a model and must not touch the GPU const bool is_router_server = params.model.path.empty() && params.model.hf_repo.empty();