diff --git a/common/arg.cpp b/common/arg.cpp index 1efbeb821..3dbe03e72 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -343,7 +343,7 @@ static handle_model_result common_params_handle_model( if (model.path.empty()) { auto auto_detected = common_get_hf_file(model.hf_repo, bearer_token, offline); if (auto_detected.repo.empty() || auto_detected.ggufFile.empty()) { - exit(1); // built without CURL, error message already printed + exit(1); // error message already printed } model.name = model.hf_repo; // repo name with tag model.hf_repo = auto_detected.repo; // repo name without tag diff --git a/common/debug.cpp b/common/debug.cpp new file mode 100644 index 000000000..fdaddb144 --- /dev/null +++ b/common/debug.cpp @@ -0,0 +1,165 @@ +#include "debug.h" + +#include "log.h" + +#include +#include + +static std::string common_ggml_ne_string(const ggml_tensor * t) { + std::string str; + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + str += std::to_string(t->ne[i]); + if (i + 1 < GGML_MAX_DIMS) { + str += ", "; + } + } + return str; +} + +static float common_ggml_get_float_value(const uint8_t * data, + ggml_type type, + const size_t * nb, + size_t i0, + size_t i1, + size_t i2, + size_t i3) { + size_t i = i3 * nb[3] + i2 * nb[2] + i1 * nb[1] + i0 * nb[0]; + float v; + if (type == GGML_TYPE_F16) { + v = ggml_fp16_to_fp32(*(const ggml_fp16_t *) &data[i]); + } else if (type == GGML_TYPE_F32) { + v = *(const float *) &data[i]; + } else if (type == GGML_TYPE_I64) { + v = (float) *(const int64_t *) &data[i]; + } else if (type == GGML_TYPE_I32) { + v = (float) *(const int32_t *) &data[i]; + } else if (type == GGML_TYPE_I16) { + v = (float) *(const int16_t *) &data[i]; + } else if (type == GGML_TYPE_I8) { + v = (float) *(const int8_t *) &data[i]; + } else if (type == GGML_TYPE_BF16) { + v = ggml_bf16_to_fp32(*(const ggml_bf16_t *) &data[i]); + } else { + GGML_ABORT("fatal error"); + } + return v; +} + +template +void common_debug_print_tensor(uint8_t * data, ggml_type type, const int64_t * ne, const size_t * nb, int64_t n) { + GGML_ASSERT(n > 0); + float sum = 0; + for (int64_t i3 = 0; i3 < ne[3]; i3++) { + for (int64_t i2 = 0; i2 < ne[2]; i2++) { + for (int64_t i1 = 0; i1 < ne[1]; i1++) { + for (int64_t i0 = 0; i0 < ne[0]; i0++) { + const float v = common_ggml_get_float_value(data, type, nb, i0, i1, i2, i3); + sum += v; + } + } + } + } + for (int64_t i3 = 0; i3 < ne[3]; i3++) { + LOG_ERR(" [\n"); + for (int64_t i2 = 0; i2 < ne[2]; i2++) { + if (i2 == n && ne[2] > 2 * n) { + LOG_ERR(" ..., \n"); + i2 = ne[2] - n; + } + LOG_ERR(" [\n"); + for (int64_t i1 = 0; i1 < ne[1]; i1++) { + if (i1 == n && ne[1] > 2 * n) { + LOG_ERR(" ..., \n"); + i1 = ne[1] - n; + } + LOG_ERR(" ["); + for (int64_t i0 = 0; i0 < ne[0]; i0++) { + if (i0 == n && ne[0] > 2 * n) { + LOG_ERR("..., "); + i0 = ne[0] - n; + } + const float v = common_ggml_get_float_value(data, type, nb, i0, i1, i2, i3); + LOG_ERR("%12.4f", v); + if (i0 < ne[0] - 1) { + LOG_ERR(", "); + } + } + LOG_ERR("],\n"); + } + LOG_ERR(" ],\n"); + } + LOG_ERR(" ]\n"); + LOG_ERR(" sum = %f\n", sum); + } + + if constexpr (abort) { + if (std::isnan(sum)) { + LOG_ERR("encountered NaN - aborting\n"); + exit(0); + } + } +} + +/** + * GGML operations callback during the graph execution. + * + * @param t current tensor + * @param ask when ask is true, the scheduler wants to know if we are interested in data from this tensor + * if we return true, a follow-up call will be made with ask=false in which we can do the actual collection. + * see ggml_backend_sched_eval_callback + * @param user_data user data to pass at each call back + * @return true to receive data or continue the graph, false otherwise + */ +template bool common_debug_cb_eval(struct ggml_tensor * t, bool ask, void * user_data) { + auto * cb_data = (base_callback_data *) user_data; + + const struct ggml_tensor * src0 = t->src[0]; + const struct ggml_tensor * src1 = t->src[1]; + + if (ask) { + return true; // Always retrieve data + } + + bool matches_filter = cb_data->tensor_filters.empty(); + + if (!matches_filter) { + for (const auto & filter : cb_data->tensor_filters) { + if (std::regex_search(t->name, filter)) { + matches_filter = true; + break; + } + } + } + + char src1_str[128] = { 0 }; + if (src1) { + snprintf(src1_str, sizeof(src1_str), "%s{%s}", src1->name, common_ggml_ne_string(src1).c_str()); + } + + if (matches_filter) { + LOG_ERR("%s: %24s = (%s) %10s(%s{%s}, %s}) = {%s}\n", __func__, t->name, ggml_type_name(t->type), + ggml_op_desc(t), src0->name, common_ggml_ne_string(src0).c_str(), src1 ? src1_str : "", + common_ggml_ne_string(t).c_str()); + } + + const bool is_host = ggml_backend_buffer_is_host(t->buffer); + + if (!is_host) { + auto n_bytes = ggml_nbytes(t); + cb_data->data.resize(n_bytes); + ggml_backend_tensor_get(t, cb_data->data.data(), 0, n_bytes); + } + + if (!ggml_is_quantized(t->type) && matches_filter) { + uint8_t * data = is_host ? (uint8_t *) t->data : cb_data->data.data(); + common_debug_print_tensor(data, t->type, t->ne, t->nb, 3); + } + + return true; +} + +// Explicit template instantiations +template bool common_debug_cb_eval(ggml_tensor *, bool, void *); +template bool common_debug_cb_eval(ggml_tensor *, bool, void *); +template void common_debug_print_tensor(uint8_t *, ggml_type, const int64_t *, const size_t *, int64_t); +template void common_debug_print_tensor(uint8_t *, ggml_type, const int64_t *, const size_t *, int64_t); diff --git a/common/debug.h b/common/debug.h new file mode 100644 index 000000000..0c5596325 --- /dev/null +++ b/common/debug.h @@ -0,0 +1,43 @@ +#pragma once +#include "common.h" +#include +#include +#include + +// common debug functions and structs + +// Print a tensor's detailed data +// data - the tensor's data in byte format +// type - the tensor's quantization type +// ne - the tensor dimensions array +// nb - the tensor strides array +// n - the number of rows/columns to fully print +template void common_debug_print_tensor(uint8_t * data, ggml_type type, const int64_t * ne, const size_t * nb, int64_t n); + +// Intended to use as callback for ggml_backend_sched_eval_callback +// prints tensors that are processed in the computation graph +// by default prints all tensors, but can be configured by creating a `base_callback_data` instance with +// non-empty filter_patterns. See examples/debug.ccp for possible usage patterns +// The template parameter determins whether an error should be thrown whenever a NaN is encountered +// in a tensor (useful for stopping debug sessions on first erroneous tensor) +// The callback data will be passed as the third parameter (user_data) +template bool common_debug_cb_eval(struct ggml_tensor * t, bool ask, void * user_data); +struct base_callback_data { + std::vector data; + std::vector tensor_filters; + + base_callback_data() = default; + + base_callback_data(common_params & params, const std::vector & filter_patterns) { + for (const auto & pattern : filter_patterns) { + try { + std::string anchored_pattern = "^" + pattern; + tensor_filters.emplace_back(anchored_pattern, std::regex::optimize); + } catch (const std::regex_error & e) { + throw std::runtime_error("Invalid regex pattern '" + pattern + "': " + e.what()); + } + } + params.cb_eval = common_debug_cb_eval; + params.cb_eval_user_data = this; + } +}; diff --git a/common/download.cpp b/common/download.cpp index dc7d5c847..a37780421 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -19,10 +19,7 @@ #include #include -#if defined(LLAMA_USE_CURL) -#include -#include -#elif defined(LLAMA_USE_HTTPLIB) +#if defined(LLAMA_USE_HTTPLIB) #include "http.h" #endif @@ -171,336 +168,7 @@ std::pair common_download_split_repo_tag(const std::st return {hf_repo, tag}; } -#ifdef LLAMA_USE_CURL - -// -// CURL utils -// - -using curl_ptr = std::unique_ptr; - -// cannot use unique_ptr for curl_slist, because we cannot update without destroying the old one -struct curl_slist_ptr { - struct curl_slist * ptr = nullptr; - ~curl_slist_ptr() { - if (ptr) { - curl_slist_free_all(ptr); - } - } -}; - -static CURLcode common_curl_perf(CURL * curl) { - CURLcode res = curl_easy_perform(curl); - if (res != CURLE_OK) { - LOG_ERR("%s: curl_easy_perform() failed\n", __func__); - } - - return res; -} - -// Send a HEAD request to retrieve the etag and last-modified headers -struct common_load_model_from_url_headers { - std::string etag; - std::string last_modified; - std::string accept_ranges; -}; - -struct FILE_deleter { - void operator()(FILE * f) const { fclose(f); } -}; - -static size_t common_header_callback(char * buffer, size_t, size_t n_items, void * userdata) { - common_load_model_from_url_headers * headers = (common_load_model_from_url_headers *) userdata; - static std::regex header_regex("([^:]+): (.*)\r\n"); - static std::regex etag_regex("ETag", std::regex_constants::icase); - static std::regex last_modified_regex("Last-Modified", std::regex_constants::icase); - static std::regex accept_ranges_regex("Accept-Ranges", std::regex_constants::icase); - std::string header(buffer, n_items); - std::smatch match; - if (std::regex_match(header, match, header_regex)) { - const std::string & key = match[1]; - const std::string & value = match[2]; - if (std::regex_match(key, match, etag_regex)) { - headers->etag = value; - } else if (std::regex_match(key, match, last_modified_regex)) { - headers->last_modified = value; - } else if (std::regex_match(key, match, accept_ranges_regex)) { - headers->accept_ranges = value; - } - } - - return n_items; -} - -static size_t common_write_callback(void * data, size_t size, size_t nmemb, void * fd) { - return std::fwrite(data, size, nmemb, static_cast(fd)); -} - -// helper function to hide password in URL -static std::string llama_download_hide_password_in_url(const std::string & url) { - // Use regex to match and replace the user[:password]@ pattern in URLs - // Pattern: scheme://[user[:password]@]host[...] - static const std::regex url_regex(R"(^(?:[A-Za-z][A-Za-z0-9+.-]://)(?:[^/@]+@)?.$)"); - std::smatch match; - - if (std::regex_match(url, match, url_regex)) { - // match[1] = scheme (e.g., "https://") - // match[2] = user[:password]@ part - // match[3] = rest of URL (host and path) - return match[1].str() + "********@" + match[3].str(); - } - - return url; // No credentials found or malformed URL -} - -static void common_curl_easy_setopt_head(CURL * curl, const std::string & url) { - // Set the URL, allow to follow http redirection - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - -# if defined(_WIN32) - // CURLSSLOPT_NATIVE_CA tells libcurl to use standard certificate store of - // operating system. Currently implemented under MS-Windows. - curl_easy_setopt(curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA); -# endif - - curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); // will trigger the HEAD verb - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); // hide head request progress - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, common_header_callback); -} - -static void common_curl_easy_setopt_get(CURL * curl) { - curl_easy_setopt(curl, CURLOPT_NOBODY, 0L); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, common_write_callback); - - // display download progress - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); -} - -static bool common_pull_file(CURL * curl, const std::string & path_temporary) { - if (std::filesystem::exists(path_temporary)) { - const std::string partial_size = std::to_string(std::filesystem::file_size(path_temporary)); - LOG_INF("%s: server supports range requests, resuming download from byte %s\n", __func__, partial_size.c_str()); - const std::string range_str = partial_size + "-"; - curl_easy_setopt(curl, CURLOPT_RANGE, range_str.c_str()); - } - - // Always open file in append mode could be resuming - std::unique_ptr outfile(fopen(path_temporary.c_str(), "ab")); - if (!outfile) { - LOG_ERR("%s: error opening local file for writing: %s\n", __func__, path_temporary.c_str()); - return false; - } - - common_curl_easy_setopt_get(curl); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, outfile.get()); - - return common_curl_perf(curl) == CURLE_OK; -} - -static bool common_download_head(CURL * curl, - curl_slist_ptr & http_headers, - const std::string & url, - const std::string & bearer_token) { - if (!curl) { - LOG_ERR("%s: error initializing libcurl\n", __func__); - return false; - } - - http_headers.ptr = curl_slist_append(http_headers.ptr, "User-Agent: llama-cpp"); - // Check if hf-token or bearer-token was specified - if (!bearer_token.empty()) { - std::string auth_header = "Authorization: Bearer " + bearer_token; - http_headers.ptr = curl_slist_append(http_headers.ptr, auth_header.c_str()); - } - - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, http_headers.ptr); - common_curl_easy_setopt_head(curl, url); - return common_curl_perf(curl) == CURLE_OK; -} - -// download one single file from remote URL to local path -// returns status code or -1 on error -static int common_download_file_single_online(const std::string & url, - const std::string & path, - const std::string & bearer_token, - const common_header_list & custom_headers) { - static const int max_attempts = 3; - static const int retry_delay_seconds = 2; - - for (int i = 0; i < max_attempts; ++i) { - std::string etag; - - // Check if the file already exists locally - const auto file_exists = std::filesystem::exists(path); - if (file_exists) { - etag = read_etag(path); - } else { - LOG_INF("%s: no previous model file found %s\n", __func__, path.c_str()); - } - - bool head_request_ok = false; - bool should_download = !file_exists; // by default, we should download if the file does not exist - - // Initialize libcurl - curl_ptr curl(curl_easy_init(), &curl_easy_cleanup); - common_load_model_from_url_headers headers; - curl_easy_setopt(curl.get(), CURLOPT_HEADERDATA, &headers); - curl_slist_ptr http_headers; - - for (const auto & h : custom_headers) { - std::string s = h.first + ": " + h.second; - http_headers.ptr = curl_slist_append(http_headers.ptr, s.c_str()); - } - const bool was_perform_successful = common_download_head(curl.get(), http_headers, url, bearer_token); - if (!was_perform_successful) { - head_request_ok = false; - } - - long http_code = 0; - curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &http_code); - if (http_code == 200) { - head_request_ok = true; - } else { - LOG_WRN("%s: HEAD invalid http status code received: %ld\n", __func__, http_code); - head_request_ok = false; - } - - // if head_request_ok is false, we don't have the etag or last-modified headers - // we leave should_download as-is, which is true if the file does not exist - bool should_download_from_scratch = false; - if (head_request_ok) { - // check if ETag or Last-Modified headers are different - // if it is, we need to download the file again - if (!etag.empty() && etag != headers.etag) { - LOG_WRN("%s: ETag header is different (%s != %s): triggering a new download\n", __func__, etag.c_str(), - headers.etag.c_str()); - should_download = true; - should_download_from_scratch = true; - } - } - - const bool accept_ranges_supported = !headers.accept_ranges.empty() && headers.accept_ranges != "none"; - if (should_download) { - if (file_exists && - !accept_ranges_supported) { // Resumable downloads not supported, delete and start again. - LOG_WRN("%s: deleting previous downloaded file: %s\n", __func__, path.c_str()); - if (remove(path.c_str()) != 0) { - LOG_ERR("%s: unable to delete file: %s\n", __func__, path.c_str()); - return -1; - } - } - - const std::string path_temporary = path + ".downloadInProgress"; - if (should_download_from_scratch) { - if (std::filesystem::exists(path_temporary)) { - if (remove(path_temporary.c_str()) != 0) { - LOG_ERR("%s: unable to delete file: %s\n", __func__, path_temporary.c_str()); - return -1; - } - } - - if (std::filesystem::exists(path)) { - if (remove(path.c_str()) != 0) { - LOG_ERR("%s: unable to delete file: %s\n", __func__, path.c_str()); - return -1; - } - } - } - if (head_request_ok) { - write_etag(path, headers.etag); - } - - // start the download - LOG_INF("%s: trying to download model from %s to %s (server_etag:%s, server_last_modified:%s)...\n", - __func__, llama_download_hide_password_in_url(url).c_str(), path_temporary.c_str(), - headers.etag.c_str(), headers.last_modified.c_str()); - const bool was_pull_successful = common_pull_file(curl.get(), path_temporary); - if (!was_pull_successful) { - if (i + 1 < max_attempts) { - const int exponential_backoff_delay = std::pow(retry_delay_seconds, i) * 1000; - LOG_WRN("%s: retrying after %d milliseconds...\n", __func__, exponential_backoff_delay); - std::this_thread::sleep_for(std::chrono::milliseconds(exponential_backoff_delay)); - } else { - LOG_ERR("%s: curl_easy_perform() failed after %d attempts\n", __func__, max_attempts); - } - - continue; - } - - long http_code = 0; - curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &http_code); - - int status = static_cast(http_code); - if (!is_http_status_ok(http_code)) { - LOG_ERR("%s: invalid http status code received: %ld\n", __func__, http_code); - return status; // TODO: maybe only return on certain codes - } - - if (rename(path_temporary.c_str(), path.c_str()) != 0) { - LOG_ERR("%s: unable to rename file: %s to %s\n", __func__, path_temporary.c_str(), path.c_str()); - return -1; - } - - return static_cast(http_code); - } else { - LOG_INF("%s: using cached file: %s\n", __func__, path.c_str()); - - return 304; // Not Modified - fake cached response - } - } - - return -1; // max attempts reached -} - -std::pair> common_remote_get_content(const std::string & url, const common_remote_params & params) { - curl_ptr curl(curl_easy_init(), &curl_easy_cleanup); - curl_slist_ptr http_headers; - std::vector res_buffer; - - curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl.get(), CURLOPT_NOPROGRESS, 1L); - curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl.get(), CURLOPT_VERBOSE, 0L); - typedef size_t(*CURLOPT_WRITEFUNCTION_PTR)(void * ptr, size_t size, size_t nmemb, void * data); - auto write_callback = [](void * ptr, size_t size, size_t nmemb, void * data) -> size_t { - auto data_vec = static_cast *>(data); - data_vec->insert(data_vec->end(), (char *)ptr, (char *)ptr + size * nmemb); - return size * nmemb; - }; - curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, static_cast(write_callback)); - curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &res_buffer); -#if defined(_WIN32) - curl_easy_setopt(curl.get(), CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA); -#endif - if (params.timeout > 0) { - curl_easy_setopt(curl.get(), CURLOPT_TIMEOUT, params.timeout); - } - if (params.max_size > 0) { - curl_easy_setopt(curl.get(), CURLOPT_MAXFILESIZE, params.max_size); - } - http_headers.ptr = curl_slist_append(http_headers.ptr, "User-Agent: llama-cpp"); - - for (const auto & header : params.headers) { - std::string header_ = header.first + ": " + header.second; - http_headers.ptr = curl_slist_append(http_headers.ptr, header_.c_str()); - } - curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, http_headers.ptr); - - CURLcode res = curl_easy_perform(curl.get()); - - if (res != CURLE_OK) { - std::string error_msg = curl_easy_strerror(res); - throw std::runtime_error("error: cannot make GET request: " + error_msg); - } - - long res_code; - curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &res_code); - - return { res_code, std::move(res_buffer) }; -} - -#elif defined(LLAMA_USE_HTTPLIB) +#if defined(LLAMA_USE_HTTPLIB) class ProgressBar { static inline std::mutex mutex; @@ -797,10 +465,6 @@ std::pair> common_remote_get_content(const std::string return { res->status, std::move(buf) }; } -#endif // LLAMA_USE_CURL - -#if defined(LLAMA_USE_CURL) || defined(LLAMA_USE_HTTPLIB) - int common_download_file_single(const std::string & url, const std::string & path, const std::string & bearer_token, @@ -1151,7 +815,7 @@ int common_download_file_single(const std::string &, throw std::runtime_error("download functionality is not enabled in this build"); } -#endif // LLAMA_USE_CURL || LLAMA_USE_HTTPLIB +#endif // defined(LLAMA_USE_HTTPLIB) std::vector common_list_cached_models() { std::vector models; diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index be83e3108..464ecbaab 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -8772,11 +8772,7 @@ class ExaoneMoEModel(Exaone4Model): self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"]) n_dense_layer = self.hparams.get("first_k_dense_replace", self.hparams.get("first_last_k_dense_replace", 0)) self.gguf_writer.add_leading_dense_block_count(n_dense_layer) - # For here, we hard-code the number of NextN/MTP layers to 1 for K-EXAONE, - # so that we can convert MTP weights to GGUF format for speculative decoding. - # This is because HF config of K-EXAONE does not have `num_nextn_predict_layers` at now. - # Will be updated when HF config is updated. - self.gguf_writer.add_nextn_predict_layers(self.hparams.get("num_nextn_predict_layers", 1)) + self.gguf_writer.add_nextn_predict_layers(self.hparams.get("num_nextn_predict_layers", 0)) self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE) diff --git a/embd_res/kcpp_sdui.embd b/embd_res/kcpp_sdui.embd index 436fb20d7..68cebafe4 100644 --- a/embd_res/kcpp_sdui.embd +++ b/embd_res/kcpp_sdui.embd @@ -5,18 +5,18 @@ Stable UI for KoboldCpp - diff --git a/embd_res/klite.embd b/embd_res/klite.embd index fe74d483e..4f126bf11 100644 --- a/embd_res/klite.embd +++ b/embd_res/klite.embd @@ -12,7 +12,7 @@ Current version indicated by LITEVER below. -->