mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-19 01:05:09 +02:00
Merge commit '961e4b26a7dd0e01e20599b27d709a74788ecb55' into concedo_experimental
# Conflicts: # .github/workflows/hip-quality-check.yml # AGENTS.md # ggml/src/ggml-hexagon/ggml-hexagon.cpp # ggml/src/ggml-hexagon/htp/CMakeLists.txt # ggml/src/ggml-hexagon/htp/concat-ops.c # ggml/src/ggml-hexagon/htp/flash-attn-ops.c # ggml/src/ggml-hexagon/htp/flash-attn-ops.h # ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h # ggml/src/ggml-hexagon/htp/hmx-queue.c # ggml/src/ggml-hexagon/htp/hmx-queue.h # ggml/src/ggml-hexagon/htp/hmx-utils.h # ggml/src/ggml-hexagon/htp/htp-ctx.h # ggml/src/ggml-hexagon/htp/hvx-utils.h # ggml/src/ggml-hexagon/htp/main.c # ggml/src/ggml-hexagon/htp/matmul-ops.c # ggml/src/ggml-hexagon/htp/rope-ops.c # ggml/src/ggml-hexagon/htp/unary-ops.c # ggml/src/ggml-hexagon/htp/worker-pool.c # ggml/src/ggml-hexagon/htp/worker-pool.h # ggml/src/ggml-hip/CMakeLists.txt # ggml/src/ggml-opencl/ggml-opencl.cpp # ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl # ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl # ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl # ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl # ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl # ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl # ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl # ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl # ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl # ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl # ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl # ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp # ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl # tests/CMakeLists.txt # tests/test-backend-ops.cpp # tools/cli/CMakeLists.txt # tools/cli/cli.cpp # tools/llama-bench/llama-bench.cpp
This commit is contained in:
Concedo
@@ -0,0 +1,130 @@
|
||||
#include "cli-client.h"
|
||||
|
||||
#include "http.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <thread>
|
||||
|
||||
// generation can stall for a long time during prompt processing, so the
|
||||
// read timeout must be generous
|
||||
static constexpr time_t CLI_HTTP_READ_TIMEOUT_SEC = 3600;
|
||||
|
||||
// upper bound for the accumulated response body kept for error reporting
|
||||
static constexpr size_t CLI_HTTP_MAX_ERROR_BODY = 1024 * 1024;
|
||||
|
||||
// returns the path with the base url's path prefix prepended (if any)
|
||||
static std::string join_path(const common_http_url & parts, const std::string & path) {
|
||||
if (parts.path.empty() || parts.path == "/") {
|
||||
return path;
|
||||
}
|
||||
std::string prefix = parts.path;
|
||||
if (prefix.back() == '/') {
|
||||
prefix.pop_back();
|
||||
}
|
||||
return prefix + path;
|
||||
}
|
||||
|
||||
std::string cli_client::get(const std::string & path) {
|
||||
auto [cli, parts] = common_http_client(server_base);
|
||||
cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);
|
||||
auto path_with_model = path + (model.empty() ? "" : ("?model=" + model));
|
||||
auto res = cli.Get(join_path(parts, path_with_model));
|
||||
if (!res) {
|
||||
throw std::runtime_error("failed to connect to " + server_base + ": " + httplib::to_string(res.error()));
|
||||
}
|
||||
if (res->status < 200 || res->status >= 300) {
|
||||
throw std::runtime_error("GET " + path + " failed with status " + std::to_string(res->status) + ": " + res->body);
|
||||
}
|
||||
return res->body;
|
||||
}
|
||||
|
||||
std::string cli_client::post(const std::string & path, const std::string & body) {
|
||||
auto [cli, parts] = common_http_client(server_base);
|
||||
cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);
|
||||
auto res = cli.Post(join_path(parts, path), body, "application/json");
|
||||
if (!res) {
|
||||
throw std::runtime_error("failed to connect to " + server_base + ": " + httplib::to_string(res.error()));
|
||||
}
|
||||
if (res->status < 200 || res->status >= 300) {
|
||||
throw std::runtime_error("POST " + path + " failed with status " + std::to_string(res->status) + ": " + res->body);
|
||||
}
|
||||
return res->body;
|
||||
}
|
||||
|
||||
std::string cli_client::post_sse(const std::string & path,
|
||||
const std::string & body,
|
||||
const std::function<bool()> & should_stop,
|
||||
const std::function<void(const std::string &)> & on_data) {
|
||||
auto [cli, parts] = common_http_client(server_base);
|
||||
cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);
|
||||
|
||||
std::string pending; // buffer for incomplete SSE lines
|
||||
std::string raw_body; // accumulated body, used only for error reporting
|
||||
|
||||
auto receiver = [&](const char * data, size_t len) -> bool {
|
||||
if (should_stop()) {
|
||||
return false; // aborts the request
|
||||
}
|
||||
if (raw_body.size() < CLI_HTTP_MAX_ERROR_BODY) {
|
||||
raw_body.append(data, std::min(len, CLI_HTTP_MAX_ERROR_BODY - raw_body.size()));
|
||||
}
|
||||
pending.append(data, len);
|
||||
size_t pos;
|
||||
while ((pos = pending.find('\n')) != std::string::npos) {
|
||||
std::string line = pending.substr(0, pos);
|
||||
pending.erase(0, pos + 1);
|
||||
if (!line.empty() && line.back() == '\r') {
|
||||
line.pop_back();
|
||||
}
|
||||
if (line.rfind("data: ", 0) != 0) {
|
||||
continue;
|
||||
}
|
||||
std::string payload = line.substr(6);
|
||||
if (payload == "[DONE]") {
|
||||
continue;
|
||||
}
|
||||
on_data(payload);
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
httplib::Headers headers = {{"Accept", "text/event-stream"}};
|
||||
auto res = cli.Post(join_path(parts, path), headers, body, "application/json", receiver);
|
||||
|
||||
if (!res) {
|
||||
if (res.error() == httplib::Error::Canceled && should_stop()) {
|
||||
return ""; // cancelled by the user
|
||||
}
|
||||
return "failed to connect to " + server_base + ": " + httplib::to_string(res.error());
|
||||
}
|
||||
if (res->status < 200 || res->status >= 300) {
|
||||
if (!raw_body.empty()) {
|
||||
return raw_body;
|
||||
}
|
||||
return "request failed with status " + std::to_string(res->status);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
bool cli_client::wait_health(const std::function<bool()> & is_aborted) {
|
||||
int connect_attempts = 0;
|
||||
while (!is_aborted()) {
|
||||
auto [cli, parts] = common_http_client(server_base);
|
||||
cli.set_connection_timeout(1, 0);
|
||||
auto res = cli.Get(join_path(parts, "/health"));
|
||||
if (res) {
|
||||
if (res->status == 200) {
|
||||
return true;
|
||||
}
|
||||
// any other status means the server is up but not ready yet
|
||||
// (e.g. 503 while the model is still loading)
|
||||
} else if (++connect_attempts >= 10) {
|
||||
last_error = "failed to connect to " + server_base + ": " + httplib::to_string(res.error());
|
||||
return false;
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(300));
|
||||
}
|
||||
last_error = "aborted while waiting for the server to become ready";
|
||||
return false;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
#pragma once
|
||||
|
||||
#include <functional>
|
||||
#include <string>
|
||||
|
||||
// openai-like client for CLI
|
||||
struct cli_client {
|
||||
std::string server_base; // base url, for example "http://127.0.0.1:8080"
|
||||
std::string last_error; // set when wait_health() fails
|
||||
|
||||
std::string model; // optional, set when the server has multiple models (router mode)
|
||||
|
||||
// simple GET request, returns the raw response body
|
||||
// throws std::runtime_error on transport error or non-2xx status
|
||||
std::string get(const std::string & path);
|
||||
|
||||
// simple POST request, returns the raw response body
|
||||
// throws std::runtime_error on transport error or non-2xx status
|
||||
std::string post(const std::string & path, const std::string & body);
|
||||
|
||||
// POST request with an SSE streaming response
|
||||
// on_data is invoked per "data:" event with the raw event payload
|
||||
// returns after the stream is finished (empty string on graceful exit)
|
||||
// otherwise, the raw error response body
|
||||
std::string post_sse(const std::string & path,
|
||||
const std::string & body,
|
||||
const std::function<bool()> & should_stop,
|
||||
const std::function<void(const std::string &)> & on_data);
|
||||
|
||||
// poll /health until the server is ready to accept requests
|
||||
// returns false if is_aborted returned true or the server is unreachable
|
||||
bool wait_health(const std::function<bool()> & is_aborted);
|
||||
};
|
||||
@@ -0,0 +1,664 @@
|
||||
#include "cli-context.h"
|
||||
#include "cli-ui.h"
|
||||
|
||||
#include "arg.h"
|
||||
#include "base64.hpp"
|
||||
#include "log.h"
|
||||
#include "console.h"
|
||||
|
||||
#define JSON_ASSERT GGML_ASSERT
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <set>
|
||||
|
||||
using json = nlohmann::ordered_json;
|
||||
|
||||
struct cli_context_impl {
|
||||
json messages = json::array();
|
||||
json pending_media = json::array(); // staged multimodal content parts
|
||||
};
|
||||
|
||||
cli_context::cli_context(const common_params & params) : params(params), impl(new cli_context_impl()) {}
|
||||
|
||||
cli_context::~cli_context() {
|
||||
shutdown();
|
||||
}
|
||||
|
||||
std::atomic<bool> & cli_context::interrupted() {
|
||||
static std::atomic<bool> flag = false;
|
||||
return flag;
|
||||
}
|
||||
|
||||
static bool should_stop() {
|
||||
return cli_context::interrupted().load();
|
||||
}
|
||||
|
||||
static constexpr size_t FILE_GLOB_MAX_RESULTS = 100;
|
||||
|
||||
const char * LLAMA_ASCII_LOGO = R"(
|
||||
▄▄ ▄▄
|
||||
██ ██
|
||||
██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄
|
||||
██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██
|
||||
██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
|
||||
██ ██
|
||||
▀▀ ▀▀
|
||||
)";
|
||||
|
||||
// number of values an arg consumes on the command line
|
||||
static int arg_num_values(const common_arg & opt) {
|
||||
if (opt.value_hint_2 != nullptr) {
|
||||
return 2;
|
||||
}
|
||||
if (opt.value_hint != nullptr) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static std::string format_error_message(const json & err) {
|
||||
if (err.contains("error") && err.at("error").is_object()) {
|
||||
const auto & e = err.at("error");
|
||||
if (e.contains("message") && e.at("message").is_string()) {
|
||||
return e.at("message").get<std::string>();
|
||||
}
|
||||
}
|
||||
return err.dump();
|
||||
}
|
||||
|
||||
// err is the raw response body of a failed request; it may or may not be JSON
|
||||
static std::string format_error_message(const std::string & err) {
|
||||
json parsed = json::parse(err, nullptr, false);
|
||||
if (!parsed.is_discarded()) {
|
||||
return format_error_message(parsed);
|
||||
}
|
||||
return err;
|
||||
}
|
||||
|
||||
static std::string media_type_from_ext(const std::string & fname) {
|
||||
std::string ext = std::filesystem::path(fname).extension().string();
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return std::tolower(c); });
|
||||
if (ext == ".wav" || ext == ".mp3") {
|
||||
return "audio";
|
||||
}
|
||||
if (ext == ".mp4" || ext == ".avi" || ext == ".mkv" || ext == ".mov" || ext == ".webm") {
|
||||
return "video";
|
||||
}
|
||||
return "image";
|
||||
}
|
||||
|
||||
bool cli_context::init() {
|
||||
ui::init(params);
|
||||
|
||||
std::optional<ui::spinner> spinner;
|
||||
|
||||
bool use_external_server = !params.server_base.empty();
|
||||
if (use_external_server) {
|
||||
std::string base = params.server_base;
|
||||
while (!base.empty() && base.back() == '/') {
|
||||
base.pop_back();
|
||||
}
|
||||
client.server_base = base;
|
||||
|
||||
spinner.emplace("Connecting to server at " + base);
|
||||
} else {
|
||||
if (params.model.path.empty() && params.model.url.empty() &&
|
||||
params.model.hf_repo.empty() && params.model.docker_repo.empty()) {
|
||||
ui::show_error(
|
||||
"no model specified",
|
||||
"use -m <file.gguf> or -hf <user/repo> to run a local model,\n"
|
||||
"or --server-base <url> to connect to a running llama-server"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
spinner.emplace("\n\nLoading model...");
|
||||
|
||||
server.emplace();
|
||||
if (!server->start(params)) {
|
||||
ui::show_error("server start failed");
|
||||
return false;
|
||||
}
|
||||
if (!server->wait_ready(should_stop)) {
|
||||
if (!should_stop()) {
|
||||
ui::show_error("the server exited before becoming ready");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
client.server_base = server->address();
|
||||
}
|
||||
|
||||
// for --server-base this is the main availability check; for a spawned
|
||||
// server it is a cheap sanity check on top of the ready signal
|
||||
auto is_aborted = [this]() {
|
||||
return should_stop() || (server && !server->alive());
|
||||
};
|
||||
bool healthy = false;
|
||||
try {
|
||||
healthy = client.wait_health(is_aborted);
|
||||
} catch (const std::exception & e) {
|
||||
client.last_error = e.what();
|
||||
}
|
||||
if (!healthy) {
|
||||
if (!should_stop()) {
|
||||
ui::show_error(client.last_error);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
if (use_external_server) {
|
||||
spinner.reset();
|
||||
if (!list_and_ask_models()) {
|
||||
return false;
|
||||
}
|
||||
// restore the spinner for the next step
|
||||
spinner.emplace("Waiting for server...");
|
||||
}
|
||||
|
||||
fetch_server_props();
|
||||
|
||||
if (!params.out_file.empty()) {
|
||||
output_file.emplace(params.out_file);
|
||||
if (!output_file->is_open()) {
|
||||
ui::show_error(string_format("failed to open output file '%s'", params.out_file.c_str()));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void cli_context::fetch_server_props() {
|
||||
try {
|
||||
json props = json::parse(client.get("/props"));
|
||||
model_name = props.value("model_alias", "");
|
||||
if (model_name.empty()) {
|
||||
const std::string path = props.value("model_path", "");
|
||||
if (!path.empty()) {
|
||||
model_name = std::filesystem::path(path).filename().string();
|
||||
}
|
||||
}
|
||||
model_ftype = props.value("model_ftype", "");
|
||||
build_info = props.value("build_info", "");
|
||||
if (props.contains("modalities") && props.at("modalities").is_object()) {
|
||||
const auto & modalities = props.at("modalities");
|
||||
has_vision = modalities.value("vision", false);
|
||||
has_audio = modalities.value("audio", false);
|
||||
has_video = modalities.value("video", false);
|
||||
}
|
||||
} catch (const std::exception & e) {
|
||||
// /props can be disabled on remote servers; not fatal
|
||||
LOG_DBG("failed to fetch /props: %s\n", e.what());
|
||||
}
|
||||
}
|
||||
|
||||
bool cli_context::list_and_ask_models() {
|
||||
json resp = json::parse(client.get("/v1/models"));
|
||||
if (!resp.contains("data") || !resp.at("data").is_array()) {
|
||||
throw std::runtime_error("invalid response from /v1/models");
|
||||
}
|
||||
std::vector<std::string> models;
|
||||
std::vector<std::string> models_display;
|
||||
for (const auto & m : resp.at("data")) {
|
||||
if (!m.contains("id") || !m.at("id").is_string()) {
|
||||
continue;
|
||||
}
|
||||
std::string name = m.at("id").get<std::string>();
|
||||
std::string display = name;
|
||||
if (m.contains("aliases") && m.at("aliases").is_array()) {
|
||||
std::vector<std::string> aliases;
|
||||
for (const auto & a : m.at("aliases")) {
|
||||
if (a.is_string()) {
|
||||
aliases.push_back(a.get<std::string>());
|
||||
}
|
||||
}
|
||||
if (!aliases.empty()) {
|
||||
display += " (" + string_join(aliases, ", ") + ")";
|
||||
}
|
||||
}
|
||||
models.push_back(name);
|
||||
models_display.push_back(display);
|
||||
}
|
||||
|
||||
// only one model: use it without asking
|
||||
if (models.size() == 1) {
|
||||
model_name = models[0];
|
||||
client.model = model_name;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string message = "\nAvailable models:";
|
||||
for (size_t i = 0; i < models_display.size(); ++i) {
|
||||
message += "\n " + std::to_string(i + 1) + ". " + models_display[i];
|
||||
}
|
||||
message += "\n";
|
||||
ui::show_message(message);
|
||||
std::string selection;
|
||||
while (selection.empty()) {
|
||||
if (should_stop()) {
|
||||
return false;
|
||||
}
|
||||
ui::user_turn user_turn;
|
||||
selection = user_turn.read_input(false, "Select model by number: ");
|
||||
if (selection.empty()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
size_t idx = std::stoul(selection);
|
||||
if (idx > 0 && idx <= models.size()) {
|
||||
model_name = models[idx - 1];
|
||||
client.model = model_name;
|
||||
ui::show_message("Selected model: " + model_name);
|
||||
break;
|
||||
}
|
||||
} catch (...) {
|
||||
// ignore
|
||||
}
|
||||
ui::show_error("Invalid selection. Please enter a valid number.");
|
||||
selection.clear();
|
||||
continue;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void cli_context::add_system_prompt() {
|
||||
if (!params.system_prompt.empty()) {
|
||||
impl->messages.push_back({
|
||||
{"role", "system"},
|
||||
{"content", params.system_prompt}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void cli_context::push_user_message(const std::string & text) {
|
||||
json content;
|
||||
if (impl->pending_media.empty()) {
|
||||
content = text;
|
||||
} else {
|
||||
// multimodal message: media parts first, then the text
|
||||
content = impl->pending_media;
|
||||
content.push_back({
|
||||
{"type", "text"},
|
||||
{"text", text}
|
||||
});
|
||||
impl->pending_media = json::array();
|
||||
}
|
||||
impl->messages.push_back({
|
||||
{"role", "user"},
|
||||
{"content", content}
|
||||
});
|
||||
}
|
||||
|
||||
bool cli_context::stage_media_file(const std::string & fname, const std::string & type) {
|
||||
std::ifstream file(fname, std::ios::binary);
|
||||
if (!file) {
|
||||
return false;
|
||||
}
|
||||
std::string data((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
|
||||
std::string encoded = base64::encode(data);
|
||||
|
||||
if (type == "audio") {
|
||||
std::string ext = std::filesystem::path(fname).extension().string();
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return std::tolower(c); });
|
||||
impl->pending_media.push_back({
|
||||
{"type", "input_audio"},
|
||||
{"input_audio", {
|
||||
{"data", encoded},
|
||||
{"format", ext == ".mp3" ? "mp3" : "wav"}
|
||||
}}
|
||||
});
|
||||
} else if (type == "video") {
|
||||
impl->pending_media.push_back({
|
||||
{"type", "input_video"},
|
||||
{"input_video", {
|
||||
{"data", encoded}
|
||||
}}
|
||||
});
|
||||
} else {
|
||||
// the server detects the actual image type from the data
|
||||
impl->pending_media.push_back({
|
||||
{"type", "image_url"},
|
||||
{"image_url", {
|
||||
{"url", "data:image/unknown;base64," + encoded}
|
||||
}}
|
||||
});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void cli_context::write_output_file(const std::string & content) {
|
||||
if (output_file) {
|
||||
(*output_file) << content;
|
||||
output_file->flush();
|
||||
}
|
||||
}
|
||||
|
||||
bool cli_context::generate_completion(generated_content & content_out, cli_timings & timings) {
|
||||
json body = {
|
||||
{"messages", impl->messages},
|
||||
{"stream", true},
|
||||
// in order to get timings even when we cancel mid-way
|
||||
{"timings_per_token", true},
|
||||
};
|
||||
if (!client.model.empty()) {
|
||||
body["model"] = client.model;
|
||||
}
|
||||
|
||||
bool stream_error = false;
|
||||
|
||||
ui::assistant_turn a;
|
||||
|
||||
std::string err = client.post_sse("/v1/chat/completions", body.dump(), should_stop, [&](const std::string & payload) {
|
||||
json chunk = json::parse(payload, nullptr, false);
|
||||
if (chunk.is_discarded()) {
|
||||
return;
|
||||
}
|
||||
if (chunk.contains("error")) {
|
||||
stream_error = true;
|
||||
ui::show_error(format_error_message(chunk));
|
||||
return;
|
||||
}
|
||||
if (chunk.contains("timings")) {
|
||||
const auto & t = chunk.at("timings");
|
||||
timings.prompt_per_second = t.value("prompt_per_second", 0.0);
|
||||
timings.predicted_per_second = t.value("predicted_per_second", 0.0);
|
||||
}
|
||||
if (!chunk.contains("choices") || !chunk.at("choices").is_array() || chunk.at("choices").empty()) {
|
||||
return;
|
||||
}
|
||||
const auto & choice = chunk.at("choices").at(0);
|
||||
if (!choice.contains("delta")) {
|
||||
return;
|
||||
}
|
||||
const auto & delta = choice.at("delta");
|
||||
if (delta.contains("reasoning_content") && delta.at("reasoning_content").is_string()) {
|
||||
const std::string text = delta.at("reasoning_content").get<std::string>();
|
||||
if (!text.empty()) {
|
||||
content_out.reasoning += text;
|
||||
a.push(ui::ASSISTANT_DISPLAY_MODE_REASONING, text);
|
||||
}
|
||||
}
|
||||
if (delta.contains("content") && delta.at("content").is_string()) {
|
||||
const std::string text = delta.at("content").get<std::string>();
|
||||
if (!text.empty()) {
|
||||
content_out.content += text;
|
||||
a.push(ui::ASSISTANT_DISPLAY_MODE_CONTENT, text);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
cli_context::interrupted().store(false);
|
||||
|
||||
if (!err.empty()) {
|
||||
ui::show_error(format_error_message(err));
|
||||
return false;
|
||||
}
|
||||
return !stream_error;
|
||||
}
|
||||
|
||||
int cli_context::run() {
|
||||
add_system_prompt();
|
||||
|
||||
std::string modalities = "text";
|
||||
if (has_vision) {
|
||||
modalities += ", vision";
|
||||
}
|
||||
if (has_audio) {
|
||||
modalities += ", audio";
|
||||
}
|
||||
if (has_video) {
|
||||
modalities += ", video";
|
||||
}
|
||||
|
||||
std::string banner;
|
||||
banner += "\n";
|
||||
banner += LLAMA_ASCII_LOGO;
|
||||
banner += "\n";
|
||||
banner += "build : " + build_info + "\n";
|
||||
banner += "model : " + model_name + "\n";
|
||||
if (!model_ftype.empty()) {
|
||||
banner += "ftype : " + model_ftype + "\n";
|
||||
}
|
||||
banner += "modalities : " + modalities + "\n";
|
||||
if (!params.system_prompt.empty()) {
|
||||
banner += "using custom system prompt\n";
|
||||
}
|
||||
banner += "\n";
|
||||
banner += "available commands:\n";
|
||||
banner += " /exit or Ctrl+C stop or exit\n";
|
||||
banner += " /regen regenerate the last response\n";
|
||||
banner += " /clear clear the chat history\n";
|
||||
banner += " /read <file> add a text file\n";
|
||||
banner += " /glob <pattern> add text files using globbing pattern\n";
|
||||
if (has_vision) {
|
||||
banner += " /image <file> add an image file\n";
|
||||
}
|
||||
if (has_audio) {
|
||||
banner += " /audio <file> add an audio file\n";
|
||||
}
|
||||
if (has_video) {
|
||||
banner += " /video <file> add a video file\n";
|
||||
}
|
||||
banner += "\n";
|
||||
|
||||
ui::show_message(banner);
|
||||
|
||||
// interactive loop
|
||||
std::string cur_msg;
|
||||
|
||||
auto add_text_file = [&](const std::string & fname) -> bool {
|
||||
std::ifstream file(fname, std::ios::binary);
|
||||
if (!file) {
|
||||
ui::show_error(string_format("file does not exist or cannot be opened: '%s'", fname.c_str()));
|
||||
return false;
|
||||
}
|
||||
std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
|
||||
cur_msg += "--- File: ";
|
||||
cur_msg += fname;
|
||||
cur_msg += " ---\n";
|
||||
cur_msg += content;
|
||||
ui::show_message(string_format("Loaded text from '%s'", fname.c_str()));
|
||||
return true;
|
||||
};
|
||||
|
||||
while (true) {
|
||||
std::string buffer;
|
||||
{
|
||||
ui::user_turn user_turn;
|
||||
|
||||
if (params.prompt.empty()) {
|
||||
buffer = user_turn.read_input(params.multiline_input);
|
||||
} else {
|
||||
// process input prompt from args
|
||||
for (auto & fname : params.image) {
|
||||
if (!stage_media_file(fname, media_type_from_ext(fname))) {
|
||||
ui::show_error(string_format("file does not exist or cannot be opened: '%s'", fname.c_str()));
|
||||
break;
|
||||
}
|
||||
ui::show_message(string_format("Loaded media from '%s'", fname.c_str()));
|
||||
}
|
||||
buffer = params.prompt;
|
||||
user_turn.echo(buffer);
|
||||
params.prompt.clear(); // only use it once
|
||||
}
|
||||
}
|
||||
|
||||
if (should_stop()) {
|
||||
cli_context::interrupted().store(false);
|
||||
break;
|
||||
}
|
||||
|
||||
// remove trailing newline
|
||||
if (!buffer.empty() && buffer.back() == '\n') {
|
||||
buffer.pop_back();
|
||||
}
|
||||
|
||||
// skip empty messages
|
||||
if (buffer.empty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
bool add_user_msg = true;
|
||||
|
||||
// process commands
|
||||
if (string_starts_with(buffer, "/exit")) {
|
||||
break;
|
||||
} else if (string_starts_with(buffer, "/regen")) {
|
||||
if (impl->messages.size() >= 2) {
|
||||
size_t last_idx = impl->messages.size() - 1;
|
||||
impl->messages.erase(last_idx);
|
||||
add_user_msg = false;
|
||||
} else {
|
||||
ui::show_error("No message to regenerate.");
|
||||
continue;
|
||||
}
|
||||
} else if (string_starts_with(buffer, "/clear")) {
|
||||
impl->messages.clear();
|
||||
add_system_prompt();
|
||||
|
||||
impl->pending_media = json::array();
|
||||
ui::show_message("Chat history cleared.");
|
||||
continue;
|
||||
} else if (
|
||||
(string_starts_with(buffer, "/image ") && has_vision) ||
|
||||
(string_starts_with(buffer, "/audio ") && has_audio) ||
|
||||
(string_starts_with(buffer, "/video ") && has_video)) {
|
||||
std::string type = buffer.substr(1, 5);
|
||||
// just in case (bad copy-paste for example), we strip all trailing/leading spaces
|
||||
std::string fname = string_strip(buffer.substr(7));
|
||||
if (!stage_media_file(fname, type)) {
|
||||
ui::show_error(string_format("file does not exist or cannot be opened: '%s'", fname.c_str()));
|
||||
continue;
|
||||
}
|
||||
ui::show_message(string_format("Loaded media from '%s'", fname.c_str()));
|
||||
write_output_file(string_format("User: Added media: %s\n", fname.c_str()));
|
||||
continue;
|
||||
} else if (string_starts_with(buffer, "/read ")) {
|
||||
std::string fname = string_strip(buffer.substr(6));
|
||||
add_text_file(fname);
|
||||
write_output_file(string_format("User: Added text file: %s\n", fname.c_str()));
|
||||
continue;
|
||||
} else if (string_starts_with(buffer, "/glob ")) {
|
||||
std::error_code ec;
|
||||
size_t count = 0;
|
||||
auto curdir = std::filesystem::current_path();
|
||||
std::string pattern = string_strip(buffer.substr(6));
|
||||
std::filesystem::path rel_path;
|
||||
|
||||
auto startglob = pattern.find_first_of("![*?");
|
||||
if (startglob != std::string::npos && startglob != 0) {
|
||||
auto endpath = pattern.substr(0, startglob).find_last_of('/');
|
||||
if (endpath != std::string::npos) {
|
||||
std::string rel_pattern = pattern.substr(0, endpath);
|
||||
#if !defined(_WIN32)
|
||||
if (string_starts_with(rel_pattern, '~')) {
|
||||
const char * home = std::getenv("HOME");
|
||||
if (home && home[0]) {
|
||||
rel_pattern = home + rel_pattern.substr(1);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
rel_path = rel_pattern;
|
||||
pattern.erase(0, endpath + 1);
|
||||
curdir /= rel_path;
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto & entry : std::filesystem::recursive_directory_iterator(curdir,
|
||||
std::filesystem::directory_options::skip_permission_denied, ec)) {
|
||||
if (!entry.is_regular_file()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::string rel = std::filesystem::relative(entry.path(), curdir, ec).string();
|
||||
if (ec) {
|
||||
ec.clear();
|
||||
continue;
|
||||
}
|
||||
std::replace(rel.begin(), rel.end(), '\\', '/');
|
||||
|
||||
if (!glob_match(pattern, rel)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const std::string full_path = (curdir / rel).string();
|
||||
if (!add_text_file(full_path)) {
|
||||
continue;
|
||||
}
|
||||
write_output_file(string_format("User: Added text file: %s\n", full_path.c_str()));
|
||||
|
||||
if (++count >= FILE_GLOB_MAX_RESULTS) {
|
||||
ui::show_error(string_format("Maximum number of globbed files allowed (%zu) reached.", FILE_GLOB_MAX_RESULTS));
|
||||
break;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
} else {
|
||||
// not a command
|
||||
cur_msg += buffer;
|
||||
}
|
||||
|
||||
// generate response
|
||||
if (add_user_msg) {
|
||||
push_user_message(cur_msg);
|
||||
write_output_file(string_format("User:\n%s\n\n", cur_msg.c_str()));
|
||||
cur_msg.clear();
|
||||
}
|
||||
|
||||
cli_timings timings;
|
||||
generated_content content;
|
||||
generate_completion(content, timings);
|
||||
|
||||
impl->messages.push_back({
|
||||
{"role", "assistant"},
|
||||
{"content", content.content}
|
||||
});
|
||||
|
||||
if (output_file) {
|
||||
std::string out_content = "Assistant:\n";
|
||||
if (!content.reasoning.empty()) {
|
||||
out_content += "[Start thinking]\n\n";
|
||||
out_content += content.reasoning;
|
||||
out_content += "[End thinking]\n\n";
|
||||
}
|
||||
out_content += content.content;
|
||||
if (!out_content.empty() && out_content.back() != '\n') {
|
||||
out_content += "\n";
|
||||
}
|
||||
out_content += "\n";
|
||||
write_output_file(out_content);
|
||||
}
|
||||
|
||||
if (params.show_timings) {
|
||||
ui::show_info(string_format(
|
||||
"\n[ Prompt: %.1f t/s | Generation: %.1f t/s ]",
|
||||
timings.prompt_per_second,
|
||||
timings.predicted_per_second
|
||||
));
|
||||
}
|
||||
|
||||
if (params.single_turn) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ui::show_message("\n\nExiting...");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void cli_context::shutdown() {
|
||||
if (server) {
|
||||
server->stop();
|
||||
server.reset();
|
||||
}
|
||||
if (output_file) {
|
||||
output_file->close();
|
||||
output_file.reset();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
#pragma once
|
||||
|
||||
#include "common.h"
|
||||
|
||||
#include "cli-client.h"
|
||||
#include "cli-server.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <fstream>
|
||||
|
||||
struct cli_timings {
|
||||
double prompt_per_second = 0.0;
|
||||
double predicted_per_second = 0.0;
|
||||
};
|
||||
|
||||
struct cli_context_impl;
|
||||
|
||||
struct cli_context {
|
||||
common_params params;
|
||||
|
||||
cli_client client; // always initialized
|
||||
std::optional<cli_server> server; // only set when no --server-base is given
|
||||
|
||||
// properties of the connected server
|
||||
// will be populated by fetch_server_props()
|
||||
std::string model_name;
|
||||
std::string model_ftype;
|
||||
std::string build_info;
|
||||
bool has_vision = false;
|
||||
bool has_audio = false;
|
||||
bool has_video = false;
|
||||
|
||||
std::optional<std::ofstream> output_file;
|
||||
|
||||
cli_context(const common_params & params);
|
||||
~cli_context();
|
||||
|
||||
// connect to --server-base or spawn a local llama-server child;
|
||||
// argc/argv are needed to forward the server-relevant args to the child
|
||||
bool init();
|
||||
|
||||
// run the interactive chat loop, returns the process exit code
|
||||
int run();
|
||||
|
||||
// stop the local server child (if any)
|
||||
void shutdown();
|
||||
|
||||
// set by the SIGINT handler; cleared once the interrupt has been handled
|
||||
static std::atomic<bool> & interrupted();
|
||||
|
||||
private:
|
||||
struct generated_content {
|
||||
std::string reasoning;
|
||||
std::string content;
|
||||
};
|
||||
bool generate_completion(generated_content & content_out, cli_timings & timings);
|
||||
void fetch_server_props();
|
||||
void add_system_prompt();
|
||||
void push_user_message(const std::string & text);
|
||||
|
||||
// check if server have multiple models (router mode)
|
||||
// if yes, list them then ask; do nothing otherwise
|
||||
bool list_and_ask_models();
|
||||
|
||||
// read a file and stage it as a multimodal content part; type is one of
|
||||
// "image", "audio", "video"; returns false if the file cannot be read
|
||||
bool stage_media_file(const std::string & fname, const std::string & type);
|
||||
|
||||
// no-op if output file is not set
|
||||
void write_output_file(const std::string & content);
|
||||
|
||||
std::unique_ptr<cli_context_impl> impl;
|
||||
};
|
||||
@@ -0,0 +1,89 @@
|
||||
#pragma once
|
||||
|
||||
#include <thread>
|
||||
|
||||
#include "http.h"
|
||||
|
||||
// llama_server will be available as a dynamic library symbol
|
||||
int llama_server(common_params & params, int argc, char ** argv);
|
||||
void llama_server_terminate();
|
||||
|
||||
struct cli_server {
|
||||
std::thread th;
|
||||
int port = -1;
|
||||
std::atomic<bool> is_alive = false;
|
||||
std::atomic<bool> is_stopping = false;
|
||||
|
||||
~cli_server() {
|
||||
stop();
|
||||
}
|
||||
|
||||
void stop() {
|
||||
if (is_stopping.exchange(true)) {
|
||||
return;
|
||||
}
|
||||
if (alive()) {
|
||||
llama_server_terminate();
|
||||
}
|
||||
if (th.joinable()) {
|
||||
th.join();
|
||||
}
|
||||
}
|
||||
|
||||
// spawn llama-server in a thread and interact with it via a random port
|
||||
bool start(common_params & params) {
|
||||
port = common_http_get_free_port();
|
||||
if (port <= 0) {
|
||||
fprintf(stderr, "failed to get a free port\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
is_alive.store(true, std::memory_order_release);
|
||||
|
||||
common_params server_params = params; // copy
|
||||
server_params.port = port;
|
||||
|
||||
th = std::thread([this, server_params]() mutable {
|
||||
// argc / argv are only used in router mode, we can skip them for now
|
||||
int res = llama_server(server_params, 0, nullptr);
|
||||
if (res != 0) {
|
||||
fprintf(stderr, "llama_server exited with code %d\n", res);
|
||||
}
|
||||
is_alive.store(false, std::memory_order_release);
|
||||
});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string address() const {
|
||||
return "http://127.0.0.1:" + std::to_string(port);
|
||||
}
|
||||
|
||||
bool wait_ready(std::function<bool()> should_stop) {
|
||||
if (!alive()) {
|
||||
return false;
|
||||
}
|
||||
while (!should_stop()) {
|
||||
auto [cli, parts] = common_http_client(address());
|
||||
cli.set_connection_timeout(1, 0);
|
||||
auto res = cli.Get("/health");
|
||||
if (res) {
|
||||
if (res->status == 200) {
|
||||
return true;
|
||||
}
|
||||
// any other status means the server is up but not ready yet
|
||||
// (e.g. 503 while the model is still loading)
|
||||
}
|
||||
if (!alive()) {
|
||||
// in case server die permanently
|
||||
return false;
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(200));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool alive() const {
|
||||
return is_alive.load(std::memory_order_acquire);
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,251 @@
|
||||
#pragma once
|
||||
|
||||
#include "common.h"
|
||||
#include "console.h"
|
||||
|
||||
#include <array>
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <filesystem>
|
||||
#include <string_view>
|
||||
|
||||
// TODO?: Make this reusable, enums, docs
|
||||
static const std::array<std::string_view, 8> cmds = {
|
||||
"/audio ",
|
||||
"/clear",
|
||||
"/exit",
|
||||
"/glob ",
|
||||
"/image ",
|
||||
"/read ",
|
||||
"/regen",
|
||||
"/video ",
|
||||
};
|
||||
|
||||
static std::vector<std::pair<std::string, size_t>> auto_completion_callback(std::string_view line, size_t cursor_byte_pos) {
|
||||
std::vector<std::pair<std::string, size_t>> matches;
|
||||
std::string cmd;
|
||||
|
||||
if (line.length() > 1 && line.front() == '/' && !std::any_of(cmds.begin(), cmds.end(), [line](std::string_view prefix) {
|
||||
return string_starts_with(line, prefix);
|
||||
})) {
|
||||
auto it = cmds.begin();
|
||||
|
||||
while ((it = std::find_if(it, cmds.end(), [line](std::string_view cmd_line) {
|
||||
return string_starts_with(cmd_line, line);
|
||||
})) != cmds.end()) {
|
||||
matches.emplace_back(*it, it->length());
|
||||
++it;
|
||||
}
|
||||
} else {
|
||||
auto it = std::find_if(cmds.begin(), cmds.end(), [line](std::string_view prefix) {
|
||||
return prefix.back() == ' ' && string_starts_with(line, prefix);
|
||||
});
|
||||
|
||||
if (it != cmds.end()) {
|
||||
cmd = *it;
|
||||
}
|
||||
}
|
||||
|
||||
if (!cmd.empty() && cmd != "/glob " && line.length() >= cmd.length() && cursor_byte_pos >= cmd.length()) {
|
||||
const std::string path_prefix = std::string(line.substr(cmd.length(), cursor_byte_pos - cmd.length()));
|
||||
const std::string path_postfix = std::string(line.substr(cursor_byte_pos));
|
||||
auto cur_dir = std::filesystem::current_path();
|
||||
std::string cur_dir_str = cur_dir.string();
|
||||
std::string expanded_prefix = path_prefix;
|
||||
|
||||
#if !defined(_WIN32)
|
||||
if (string_starts_with(path_prefix, '~')) {
|
||||
const char * home = std::getenv("HOME");
|
||||
if (home && home[0]) {
|
||||
expanded_prefix = home + path_prefix.substr(1);
|
||||
}
|
||||
}
|
||||
if (string_starts_with(expanded_prefix, '/')) {
|
||||
#else
|
||||
if (std::isalpha(static_cast<unsigned char>(expanded_prefix[0])) && expanded_prefix.find(':') == 1) {
|
||||
#endif
|
||||
cur_dir = std::filesystem::path(expanded_prefix).parent_path();
|
||||
cur_dir_str.clear();
|
||||
} else if (!path_prefix.empty()) {
|
||||
cur_dir /= std::filesystem::path(path_prefix).parent_path();
|
||||
}
|
||||
|
||||
std::error_code ec;
|
||||
for (const auto & entry : std::filesystem::directory_iterator(cur_dir, ec)) {
|
||||
if (ec) {
|
||||
break;
|
||||
}
|
||||
if (!entry.exists(ec)) {
|
||||
ec.clear();
|
||||
continue;
|
||||
}
|
||||
|
||||
const std::string path_full = entry.path().string();
|
||||
std::string path_entry = !cur_dir_str.empty() && string_starts_with(path_full, cur_dir_str) ? path_full.substr(cur_dir_str.length() + 1) : path_full;
|
||||
|
||||
if (entry.is_directory(ec)) {
|
||||
path_entry.push_back(std::filesystem::path::preferred_separator);
|
||||
}
|
||||
|
||||
if (expanded_prefix.empty() || string_starts_with(path_entry, expanded_prefix)) {
|
||||
const std::string updated_line = cmd + path_entry;
|
||||
matches.emplace_back(updated_line + path_postfix, updated_line.length());
|
||||
}
|
||||
|
||||
if (ec) {
|
||||
ec.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (matches.empty()) {
|
||||
const std::string updated_line = cmd + path_prefix;
|
||||
matches.emplace_back(updated_line + path_postfix, updated_line.length());
|
||||
}
|
||||
|
||||
// Add the longest common prefix
|
||||
if (!expanded_prefix.empty() && matches.size() > 1) {
|
||||
const std::string_view match0(matches[0].first);
|
||||
const std::string_view match1(matches[1].first);
|
||||
auto it = std::mismatch(match0.begin(), match0.end(), match1.begin(), match1.end());
|
||||
size_t len = it.first - match0.begin();
|
||||
|
||||
for (size_t i = 2; i < matches.size(); ++i) {
|
||||
const std::string_view matchi(matches[i].first);
|
||||
auto cmp = std::mismatch(match0.begin(), match0.end(), matchi.begin(), matchi.end());
|
||||
len = std::min(len, static_cast<size_t>(cmp.first - match0.begin()));
|
||||
}
|
||||
|
||||
const std::string updated_line = std::string(match0.substr(0, len));
|
||||
matches.emplace_back(updated_line + path_postfix, updated_line.length());
|
||||
}
|
||||
|
||||
std::sort(matches.begin(), matches.end(), [](const auto & a, const auto & b) {
|
||||
return a.first.compare(0, a.second, b.first, 0, b.second) < 0;
|
||||
});
|
||||
}
|
||||
|
||||
return matches;
|
||||
}
|
||||
|
||||
// note: make this view implementation generic, so that we can move to TUI in the future if we want to
|
||||
namespace ui {
|
||||
static void init(const common_params & params) {
|
||||
// TODO: avoid using atexit() here by making `console` a singleton
|
||||
console::init(params.simple_io, params.use_color);
|
||||
atexit([]() { console::cleanup(); });
|
||||
|
||||
console::set_completion_callback(auto_completion_callback);
|
||||
}
|
||||
|
||||
struct spinner {
|
||||
spinner(const std::string & message) {
|
||||
if (!message.empty()) {
|
||||
console::log("%s ", message.c_str());
|
||||
}
|
||||
console::spinner::start();
|
||||
}
|
||||
~spinner() {
|
||||
console::spinner::stop();
|
||||
}
|
||||
};
|
||||
|
||||
struct user_turn {
|
||||
user_turn() {
|
||||
console::set_display(DISPLAY_TYPE_USER_INPUT);
|
||||
}
|
||||
~user_turn() {
|
||||
console::set_display(DISPLAY_TYPE_RESET);
|
||||
}
|
||||
void echo(const std::string & buffer) {
|
||||
if (buffer.size() > 500) {
|
||||
console::log("\n> %s ... (truncated)\n", buffer.substr(0, 500).c_str());
|
||||
} else {
|
||||
console::log("\n> %s\n", buffer.c_str());
|
||||
}
|
||||
}
|
||||
std::string read_input(bool multiline_input, const char * prompt = nullptr) {
|
||||
if (prompt) {
|
||||
console::log("%s", prompt);
|
||||
} else {
|
||||
console::log("\n> ");
|
||||
}
|
||||
std::string buffer;
|
||||
std::string line;
|
||||
bool another_line = true;
|
||||
do {
|
||||
another_line = console::readline(line, multiline_input);
|
||||
buffer += line;
|
||||
} while (another_line);
|
||||
return buffer;
|
||||
}
|
||||
};
|
||||
|
||||
enum assistant_display_mode {
|
||||
ASSISTANT_DISPLAY_MODE_REASONING,
|
||||
ASSISTANT_DISPLAY_MODE_CONTENT,
|
||||
};
|
||||
struct assistant_turn {
|
||||
assistant_display_mode mode = ASSISTANT_DISPLAY_MODE_CONTENT;
|
||||
bool trailing_newline = true;
|
||||
bool is_inside_reasoning = false;
|
||||
assistant_turn() {
|
||||
console::set_display(DISPLAY_TYPE_RESET);
|
||||
}
|
||||
~assistant_turn() {
|
||||
console::set_display(DISPLAY_TYPE_RESET);
|
||||
add_newline_if_needed();
|
||||
}
|
||||
void push(assistant_display_mode m, const std::string & buffer) {
|
||||
if (m != mode) {
|
||||
add_newline_if_needed();
|
||||
switch (m) {
|
||||
case ASSISTANT_DISPLAY_MODE_CONTENT:
|
||||
{
|
||||
if (is_inside_reasoning) {
|
||||
console::log("[End thinking]\n\n");
|
||||
is_inside_reasoning = false;
|
||||
}
|
||||
console::set_display(DISPLAY_TYPE_RESET);
|
||||
} break;
|
||||
case ASSISTANT_DISPLAY_MODE_REASONING:
|
||||
{
|
||||
console::set_display(DISPLAY_TYPE_REASONING);
|
||||
is_inside_reasoning = true;
|
||||
console::log("\n[Start thinking]\n\n");
|
||||
} break;
|
||||
}
|
||||
}
|
||||
mode = m;
|
||||
if (buffer.empty()) {
|
||||
return;
|
||||
}
|
||||
trailing_newline = buffer.back() == '\n';
|
||||
console::log("%s", buffer.c_str());
|
||||
console::flush();
|
||||
}
|
||||
void add_newline_if_needed() {
|
||||
if (!trailing_newline) {
|
||||
console::log("\n");
|
||||
console::flush();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
static void show_error(const std::string & title, const std::string & message = "") {
|
||||
console::spinner::stop();
|
||||
console::error("Error: %s\n", title.c_str());
|
||||
if (!message.empty()) {
|
||||
console::log("%s\n", message.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
static void show_message(const std::string & message) {
|
||||
console::log("%s\n", message.c_str());
|
||||
}
|
||||
|
||||
static void show_info(const std::string & message) {
|
||||
console::set_display(DISPLAY_TYPE_INFO);
|
||||
console::log("%s\n", message.c_str());
|
||||
console::set_display(DISPLAY_TYPE_RESET);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user