From ae5183be10de8baa16ac47d70baff39ed3ce90b1 Mon Sep 17 00:00:00 2001 From: Wagner Bruna Date: Fri, 13 Feb 2026 21:12:08 -0300 Subject: [PATCH] sd: sync to master-504-636d3cb (#1969) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * sd: sync to master-504-636d3cb * sd: fix and simplify limit calculation - restore the "arbitrarily high" 8192 limit, since it's used to turn off the img_hard_limit (and if each side was always limited by 2048, we wouldn't need hard_megapixel_res_limit) - avoid changing the config cfg_square_limit during a generation - apply the hard_megapixel_res_limit only in the configuration-changed path, since the default path uses constants - clean up comments The calculation itself remains the same: - default area limit: 832² for SD1.5/SD2, 1024² otherwise - configured limit always between 64 and 2048 --- Makefile | 2 +- otherarch/sdcpp/clip.hpp | 3 +- otherarch/sdcpp/conditioner.hpp | 375 ++++++++------ otherarch/sdcpp/ggml_extend.hpp | 186 ++++++- otherarch/sdcpp/latent-preview.h | 468 +++++++++--------- otherarch/sdcpp/llm.hpp | 5 +- otherarch/sdcpp/lora.hpp | 116 ++++- otherarch/sdcpp/main.cpp | 37 +- otherarch/sdcpp/model.cpp | 60 --- otherarch/sdcpp/model.h | 7 - otherarch/sdcpp/sdtype_adapter.cpp | 47 +- otherarch/sdcpp/stable-diffusion.cpp | 25 +- otherarch/sdcpp/t5.hpp | 5 +- otherarch/sdcpp/upscaler.cpp | 2 +- .../sdcpp/{vocab.hpp => vocab/clip_t5.hpp} | 4 +- .../{vocab_mistral.hpp => vocab/mistral.hpp} | 4 +- .../sdcpp/{vocab_qwen.hpp => vocab/qwen.hpp} | 2 +- .../sdcpp/{vocab_umt5.hpp => vocab/umt5.hpp} | 2 +- otherarch/sdcpp/vocab/vocab.cpp | 35 ++ otherarch/sdcpp/vocab/vocab.h | 13 + 20 files changed, 860 insertions(+), 538 deletions(-) rename otherarch/sdcpp/{vocab.hpp => vocab/clip_t5.hpp} (99%) rename otherarch/sdcpp/{vocab_mistral.hpp => vocab/mistral.hpp} (99%) rename otherarch/sdcpp/{vocab_qwen.hpp => vocab/qwen.hpp} (99%) rename otherarch/sdcpp/{vocab_umt5.hpp => vocab/umt5.hpp} (99%) create mode 100644 otherarch/sdcpp/vocab/vocab.cpp create mode 100644 otherarch/sdcpp/vocab/vocab.h diff --git a/Makefile b/Makefile index 52de513c9..b3687d7e0 100644 --- a/Makefile +++ b/Makefile @@ -724,7 +724,7 @@ mainvk: tools/completion/completion.cpp common/arg.cpp common/speculative.cpp co $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) fitparams: tools/fit-params/fit-params.cpp common/arg.cpp common/speculative.cpp common/ngram-cache.cpp common/ngram-map.cpp common/ngram-mod.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_vulkan.o llava.o ggml-backend_vulkan.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -sdmain: otherarch/sdcpp/util.cpp otherarch/sdcpp/main.cpp otherarch/sdcpp/stable-diffusion.cpp otherarch/sdcpp/upscaler.cpp otherarch/sdcpp/model.cpp otherarch/sdcpp/name_conversion.cpp otherarch/sdcpp/tokenize_util.cpp otherarch/sdcpp/version.cpp otherarch/sdcpp/thirdparty/zip.c build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_default.o llava.o ggml-backend_default.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +sdmain: otherarch/sdcpp/util.cpp otherarch/sdcpp/main.cpp otherarch/sdcpp/stable-diffusion.cpp otherarch/sdcpp/upscaler.cpp otherarch/sdcpp/model.cpp otherarch/sdcpp/name_conversion.cpp otherarch/sdcpp/tokenize_util.cpp otherarch/sdcpp/version.cpp otherarch/sdcpp/thirdparty/zip.c otherarch/sdcpp/vocab/vocab.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_default.o llava.o ggml-backend_default.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_default.o llava.o ggml-backend_default.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) diff --git a/otherarch/sdcpp/clip.hpp b/otherarch/sdcpp/clip.hpp index 3fc656650..adecd4d21 100644 --- a/otherarch/sdcpp/clip.hpp +++ b/otherarch/sdcpp/clip.hpp @@ -4,6 +4,7 @@ #include "ggml_extend.hpp" #include "model.h" #include "tokenize_util.h" +#include "vocab/vocab.h" /*================================================== CLIPTokenizer ===================================================*/ @@ -110,7 +111,7 @@ public: if (merges_utf8_str.size() > 0) { load_from_merges(merges_utf8_str); } else { - load_from_merges(ModelLoader::load_merges()); + load_from_merges(load_clip_merges()); } add_special_token("<|startoftext|>"); add_special_token("<|endoftext|>"); diff --git a/otherarch/sdcpp/conditioner.hpp b/otherarch/sdcpp/conditioner.hpp index b1876954f..4317ed18a 100644 --- a/otherarch/sdcpp/conditioner.hpp +++ b/otherarch/sdcpp/conditioner.hpp @@ -10,9 +10,14 @@ struct SDCondition { struct ggml_tensor* c_vector = nullptr; // aka y struct ggml_tensor* c_concat = nullptr; + std::vector extra_c_crossattns; + SDCondition() = default; - SDCondition(struct ggml_tensor* c_crossattn, struct ggml_tensor* c_vector, struct ggml_tensor* c_concat) - : c_crossattn(c_crossattn), c_vector(c_vector), c_concat(c_concat) {} + SDCondition(struct ggml_tensor* c_crossattn, + struct ggml_tensor* c_vector, + struct ggml_tensor* c_concat, + const std::vector& extra_c_crossattns = {}) + : c_crossattn(c_crossattn), c_vector(c_vector), c_concat(c_concat), extra_c_crossattns(extra_c_crossattns) {} }; struct ConditionerParams { @@ -1696,18 +1701,23 @@ struct LLMEmbedder : public Conditioner { } std::tuple, std::vector> tokenize(std::string text, - std::pair attn_range, + const std::pair& attn_range, size_t max_length = 0, bool padding = false) { std::vector> parsed_attention; - parsed_attention.emplace_back(text.substr(0, attn_range.first), 1.f); - if (attn_range.second - attn_range.first > 0) { - auto new_parsed_attention = parse_prompt_attention(text.substr(attn_range.first, attn_range.second - attn_range.first)); - parsed_attention.insert(parsed_attention.end(), - new_parsed_attention.begin(), - new_parsed_attention.end()); + if (attn_range.first >= 0 && attn_range.second > 0) { + parsed_attention.emplace_back(text.substr(0, attn_range.first), 1.f); + if (attn_range.second - attn_range.first > 0) { + auto new_parsed_attention = parse_prompt_attention(text.substr(attn_range.first, attn_range.second - attn_range.first)); + parsed_attention.insert(parsed_attention.end(), + new_parsed_attention.begin(), + new_parsed_attention.end()); + } + parsed_attention.emplace_back(text.substr(attn_range.second), 1.f); + } else { + parsed_attention.emplace_back(text, 1.f); } - parsed_attention.emplace_back(text.substr(attn_range.second), 1.f); + { std::stringstream ss; ss << "["; @@ -1738,156 +1748,27 @@ struct LLMEmbedder : public Conditioner { return {tokens, weights}; } - SDCondition get_learned_condition(ggml_context* work_ctx, - int n_threads, - const ConditionerParams& conditioner_params) override { - std::string prompt; - std::vector> image_embeds; - std::pair prompt_attn_range; - int prompt_template_encode_start_idx = 34; - int max_length = 0; - std::set out_layers; - std::vector tokens; - std::vector weights; + ggml_tensor* encode_prompt(ggml_context* work_ctx, + int n_threads, + const std::string prompt, + const std::pair& prompt_attn_range, + int max_length, + int min_length, + std::vector> image_embeds, + const std::set& out_layers, + int prompt_template_encode_start_idx) { + auto tokens_and_weights = tokenize(prompt, prompt_attn_range); + auto& tokens = std::get<0>(tokens_and_weights); + auto& weights = std::get<1>(tokens_and_weights); std::vector mask; - if (llm->enable_vision && conditioner_params.ref_images.size() > 0) { - LOG_INFO("QwenImageEditPlusPipeline"); - prompt_template_encode_start_idx = 64; - int image_embed_idx = 64 + 6; - - int min_pixels = 384 * 384; - int max_pixels = 560 * 560; - std::string placeholder = "<|image_pad|>"; - std::string img_prompt; - - for (int i = 0; i < conditioner_params.ref_images.size(); i++) { - sd_image_f32_t image = sd_image_t_to_sd_image_f32_t(*conditioner_params.ref_images[i]); - double factor = llm->params.vision.patch_size * llm->params.vision.spatial_merge_size; - int height = image.height; - int width = image.width; - int h_bar = static_cast(std::round(height / factor) * factor); - int w_bar = static_cast(std::round(width / factor) * factor); - - if (static_cast(h_bar) * w_bar > max_pixels) { - double beta = std::sqrt((height * width) / static_cast(max_pixels)); - h_bar = std::max(static_cast(factor), - static_cast(std::floor(height / beta / factor)) * static_cast(factor)); - w_bar = std::max(static_cast(factor), - static_cast(std::floor(width / beta / factor)) * static_cast(factor)); - } else if (static_cast(h_bar) * w_bar < min_pixels) { - double beta = std::sqrt(static_cast(min_pixels) / (height * width)); - h_bar = static_cast(std::ceil(height * beta / factor)) * static_cast(factor); - w_bar = static_cast(std::ceil(width * beta / factor)) * static_cast(factor); - } - - LOG_DEBUG("resize conditioner ref image %d from %dx%d to %dx%d", i, image.height, image.width, h_bar, w_bar); - - sd_image_f32_t resized_image = clip_preprocess(image, w_bar, h_bar); - free(image.data); - image.data = nullptr; - - ggml_tensor* image_tensor = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, resized_image.width, resized_image.height, 3, 1); - sd_image_f32_to_ggml_tensor(resized_image, image_tensor, false); - free(resized_image.data); - resized_image.data = nullptr; - - ggml_tensor* image_embed = nullptr; - llm->encode_image(n_threads, image_tensor, &image_embed, work_ctx); - image_embeds.emplace_back(image_embed_idx, image_embed); - image_embed_idx += 1 + static_cast(image_embed->ne[1]) + 6; - - img_prompt += "Picture " + std::to_string(i + 1) + ": <|vision_start|>"; // [24669, 220, index, 25, 220, 151652] - int64_t num_image_tokens = image_embed->ne[1]; - img_prompt.reserve(num_image_tokens * placeholder.size()); - for (int j = 0; j < num_image_tokens; j++) { - img_prompt += placeholder; - } - img_prompt += "<|vision_end|>"; - } - - prompt = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n"; - prompt += img_prompt; - - prompt_attn_range.first = static_cast(prompt.size()); - prompt += conditioner_params.text; - prompt_attn_range.second = static_cast(prompt.size()); - - prompt += "<|im_end|>\n<|im_start|>assistant\n"; - } else if (version == VERSION_FLUX2) { - prompt_template_encode_start_idx = 0; - out_layers = {10, 20, 30}; - - prompt = "[SYSTEM_PROMPT]You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object\nattribution and actions without speculation.[/SYSTEM_PROMPT][INST]"; - - prompt_attn_range.first = static_cast(prompt.size()); - prompt += conditioner_params.text; - prompt_attn_range.second = static_cast(prompt.size()); - - prompt += "[/INST]"; - } else if (sd_version_is_z_image(version)) { - prompt_template_encode_start_idx = 0; - out_layers = {35}; // -2 - - prompt = "<|im_start|>user\n"; - - prompt_attn_range.first = static_cast(prompt.size()); - prompt += conditioner_params.text; - prompt_attn_range.second = static_cast(prompt.size()); - - prompt += "<|im_end|>\n<|im_start|>assistant\n"; - } else if (version == VERSION_FLUX2_KLEIN) { - prompt_template_encode_start_idx = 0; - max_length = 512; - out_layers = {9, 18, 27}; - - prompt = "<|im_start|>user\n"; - - prompt_attn_range.first = static_cast(prompt.size()); - prompt += conditioner_params.text; - prompt_attn_range.second = static_cast(prompt.size()); - - prompt += "<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; - - auto tokens_and_weights = tokenize(prompt, prompt_attn_range, 0, false); - tokens = std::get<0>(tokens_and_weights); - weights = std::get<1>(tokens_and_weights); + if (max_length > 0 && tokens.size() < max_length) { mask.insert(mask.end(), tokens.size(), 1.f); - if (tokens.size() < max_length) { - mask.insert(mask.end(), max_length - tokens.size(), 0.f); - tokenizer->pad_tokens(tokens, weights, max_length, true); - } - } else if (version == VERSION_OVIS_IMAGE) { - prompt_template_encode_start_idx = 28; - max_length = prompt_template_encode_start_idx + 256; - - prompt = "<|im_start|>user\nDescribe the image by detailing the color, quantity, text, shape, size, texture, spatial relationships of the objects and background:"; - - prompt_attn_range.first = static_cast(prompt.size()); - prompt += " " + conditioner_params.text; - prompt_attn_range.second = static_cast(prompt.size()); - - prompt += "<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; - } else { - prompt_template_encode_start_idx = 34; - - prompt = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n"; - - prompt_attn_range.first = static_cast(prompt.size()); - prompt += conditioner_params.text; - prompt_attn_range.second = static_cast(prompt.size()); - - prompt += "<|im_end|>\n<|im_start|>assistant\n"; + mask.insert(mask.end(), max_length - tokens.size(), 0.f); + tokenizer->pad_tokens(tokens, weights, max_length, true); } - if (tokens.empty()) { - auto tokens_and_weights = tokenize(prompt, prompt_attn_range, max_length, max_length > 0); - tokens = std::get<0>(tokens_and_weights); - weights = std::get<1>(tokens_and_weights); - } - - int64_t t0 = ggml_time_ms(); - struct ggml_tensor* hidden_states = nullptr; // [N, n_token, 3584] + struct ggml_tensor* hidden_states = nullptr; // [N, n_token, hidden_size] auto input_ids = vector_to_ggml_tensor_i32(work_ctx, tokens); @@ -1930,11 +1811,6 @@ struct LLMEmbedder : public Conditioner { GGML_ASSERT(hidden_states->ne[1] > prompt_template_encode_start_idx); - int64_t min_length = 0; - if (version == VERSION_FLUX2) { - min_length = 512; - } - int64_t zero_pad_len = 0; if (min_length > 0) { if (hidden_states->ne[1] - prompt_template_encode_start_idx < min_length) { @@ -1956,11 +1832,186 @@ struct LLMEmbedder : public Conditioner { ggml_ext_tensor_set_f32(new_hidden_states, value, i0, i1, i2, i3); }); - // print_ggml_tensor(new_hidden_states); + return new_hidden_states; + } + + SDCondition get_learned_condition(ggml_context* work_ctx, + int n_threads, + const ConditionerParams& conditioner_params) override { + std::string prompt; + std::pair prompt_attn_range; + std::vector extra_prompts; + std::vector> extra_prompts_attn_range; + std::vector> image_embeds; + int prompt_template_encode_start_idx = 34; + int max_length = 0; // pad tokens + int min_length = 0; // zero pad hidden_states + std::set out_layers; + + int64_t t0 = ggml_time_ms(); + + if (sd_version_is_qwen_image(version)) { + if (llm->enable_vision && !conditioner_params.ref_images.empty()) { + LOG_INFO("QwenImageEditPlusPipeline"); + prompt_template_encode_start_idx = 64; + int image_embed_idx = 64 + 6; + + int min_pixels = 384 * 384; + int max_pixels = 560 * 560; + std::string placeholder = "<|image_pad|>"; + std::string img_prompt; + + for (int i = 0; i < conditioner_params.ref_images.size(); i++) { + sd_image_f32_t image = sd_image_t_to_sd_image_f32_t(*conditioner_params.ref_images[i]); + double factor = llm->params.vision.patch_size * llm->params.vision.spatial_merge_size; + int height = image.height; + int width = image.width; + int h_bar = static_cast(std::round(height / factor) * factor); + int w_bar = static_cast(std::round(width / factor) * factor); + + if (static_cast(h_bar) * w_bar > max_pixels) { + double beta = std::sqrt((height * width) / static_cast(max_pixels)); + h_bar = std::max(static_cast(factor), + static_cast(std::floor(height / beta / factor)) * static_cast(factor)); + w_bar = std::max(static_cast(factor), + static_cast(std::floor(width / beta / factor)) * static_cast(factor)); + } else if (static_cast(h_bar) * w_bar < min_pixels) { + double beta = std::sqrt(static_cast(min_pixels) / (height * width)); + h_bar = static_cast(std::ceil(height * beta / factor)) * static_cast(factor); + w_bar = static_cast(std::ceil(width * beta / factor)) * static_cast(factor); + } + + LOG_DEBUG("resize conditioner ref image %d from %dx%d to %dx%d", i, image.height, image.width, h_bar, w_bar); + + sd_image_f32_t resized_image = clip_preprocess(image, w_bar, h_bar); + free(image.data); + image.data = nullptr; + + ggml_tensor* image_tensor = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, resized_image.width, resized_image.height, 3, 1); + sd_image_f32_to_ggml_tensor(resized_image, image_tensor, false); + free(resized_image.data); + resized_image.data = nullptr; + + ggml_tensor* image_embed = nullptr; + llm->encode_image(n_threads, image_tensor, &image_embed, work_ctx); + image_embeds.emplace_back(image_embed_idx, image_embed); + image_embed_idx += 1 + static_cast(image_embed->ne[1]) + 6; + + img_prompt += "Picture " + std::to_string(i + 1) + ": <|vision_start|>"; // [24669, 220, index, 25, 220, 151652] + int64_t num_image_tokens = image_embed->ne[1]; + img_prompt.reserve(num_image_tokens * placeholder.size()); + for (int j = 0; j < num_image_tokens; j++) { + img_prompt += placeholder; + } + img_prompt += "<|vision_end|>"; + } + + prompt = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n"; + prompt += img_prompt; + + prompt_attn_range.first = static_cast(prompt.size()); + prompt += conditioner_params.text; + prompt_attn_range.second = static_cast(prompt.size()); + + prompt += "<|im_end|>\n<|im_start|>assistant\n"; + } else { + prompt_template_encode_start_idx = 34; + + prompt = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n"; + + prompt_attn_range.first = static_cast(prompt.size()); + prompt += conditioner_params.text; + prompt_attn_range.second = static_cast(prompt.size()); + + prompt += "<|im_end|>\n<|im_start|>assistant\n"; + } + } else if (version == VERSION_FLUX2) { + prompt_template_encode_start_idx = 0; + min_length = 512; + out_layers = {10, 20, 30}; + + prompt = "[SYSTEM_PROMPT]You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object\nattribution and actions without speculation.[/SYSTEM_PROMPT][INST]"; + + prompt_attn_range.first = static_cast(prompt.size()); + prompt += conditioner_params.text; + prompt_attn_range.second = static_cast(prompt.size()); + + prompt += "[/INST]"; + } else if (sd_version_is_z_image(version)) { + prompt_template_encode_start_idx = 0; + out_layers = {35}; // -2 + + if (!conditioner_params.ref_images.empty()) { + LOG_INFO("ZImageOmniPipeline"); + prompt = "<|im_start|>user\n<|vision_start|>"; + for (int i = 0; i < conditioner_params.ref_images.size() - 1; i++) { + extra_prompts.push_back("<|vision_end|><|vision_start|>"); + } + extra_prompts.push_back("<|vision_end|>" + conditioner_params.text + "<|im_end|>\n<|im_start|>assistant\n<|vision_start|>"); + extra_prompts.push_back("<|vision_end|><|im_end|>"); + } else { + prompt = "<|im_start|>user\n"; + + prompt_attn_range.first = static_cast(prompt.size()); + prompt += conditioner_params.text; + prompt_attn_range.second = static_cast(prompt.size()); + + prompt += "<|im_end|>\n<|im_start|>assistant\n"; + } + } else if (version == VERSION_FLUX2_KLEIN) { + prompt_template_encode_start_idx = 0; + max_length = 512; + out_layers = {9, 18, 27}; + + prompt = "<|im_start|>user\n"; + + prompt_attn_range.first = static_cast(prompt.size()); + prompt += conditioner_params.text; + prompt_attn_range.second = static_cast(prompt.size()); + + prompt += "<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; + } else if (version == VERSION_OVIS_IMAGE) { + prompt_template_encode_start_idx = 28; + max_length = prompt_template_encode_start_idx + 256; + + prompt = "<|im_start|>user\nDescribe the image by detailing the color, quantity, text, shape, size, texture, spatial relationships of the objects and background:"; + + prompt_attn_range.first = static_cast(prompt.size()); + prompt += " " + conditioner_params.text; + prompt_attn_range.second = static_cast(prompt.size()); + + prompt += "<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; + } else { + GGML_ABORT("unknown version %d", version); + } + + auto hidden_states = encode_prompt(work_ctx, + n_threads, + prompt, + prompt_attn_range, + max_length, + min_length, + image_embeds, + out_layers, + prompt_template_encode_start_idx); + + std::vector extra_hidden_states_vec; + for (int i = 0; i < extra_prompts.size(); i++) { + auto extra_hidden_states = encode_prompt(work_ctx, + n_threads, + extra_prompts[i], + extra_prompts_attn_range[i], + max_length, + min_length, + image_embeds, + out_layers, + prompt_template_encode_start_idx); + extra_hidden_states_vec.push_back(extra_hidden_states); + } int64_t t1 = ggml_time_ms(); LOG_DEBUG("computing condition graph completed, taking %" PRId64 " ms", t1 - t0); - return {new_hidden_states, nullptr, nullptr}; + return {hidden_states, nullptr, nullptr, extra_hidden_states_vec}; } }; diff --git a/otherarch/sdcpp/ggml_extend.hpp b/otherarch/sdcpp/ggml_extend.hpp index 3419fa918..cac79bb21 100644 --- a/otherarch/sdcpp/ggml_extend.hpp +++ b/otherarch/sdcpp/ggml_extend.hpp @@ -767,7 +767,7 @@ __STATIC_INLINE__ ggml_tensor* ggml_ext_silu_act(ggml_context* ctx, ggml_tensor* return x; } -typedef std::function on_tile_process; +typedef std::function on_tile_process; __STATIC_INLINE__ void sd_tiling_calc_tiles(int& num_tiles_dim, float& tile_overlap_factor_dim, @@ -918,12 +918,15 @@ __STATIC_INLINE__ void sd_tiling_non_square(ggml_tensor* input, int64_t t1 = ggml_time_ms(); ggml_ext_tensor_split_2d(input, input_tile, x_in, y_in); - on_processing(input_tile, output_tile, false); - ggml_ext_tensor_merge_2d(output_tile, output, x_out, y_out, overlap_x_out, overlap_y_out, dx, dy); + if (on_processing(input_tile, output_tile, false)) { + ggml_ext_tensor_merge_2d(output_tile, output, x_out, y_out, overlap_x_out, overlap_y_out, dx, dy); - int64_t t2 = ggml_time_ms(); - last_time = (t2 - t1) / 1000.0f; - pretty_progress(tile_count, num_tiles, last_time); + int64_t t2 = ggml_time_ms(); + last_time = (t2 - t1) / 1000.0f; + pretty_progress(tile_count, num_tiles, last_time); + } else { + LOG_ERROR("Failed to process patch %d at (%d, %d)", tile_count, x, y); + } tile_count++; } last_x = false; @@ -1577,7 +1580,7 @@ struct WeightAdapter { bool force_prec_f32 = false; float scale = 1.f; } linear; - struct { + struct conv2d_params_t { int s0 = 1; int s1 = 1; int p0 = 0; @@ -2630,4 +2633,173 @@ public: } }; +__STATIC_INLINE__ struct ggml_tensor* ggml_ext_lokr_forward( + struct ggml_context* ctx, + struct ggml_tensor* h, // Input: [q, batch] or [W, H, q, batch] + struct ggml_tensor* w1, // Outer C (Full rank) + struct ggml_tensor* w1a, // Outer A (Low rank part 1) + struct ggml_tensor* w1b, // Outer B (Low rank part 2) + struct ggml_tensor* w2, // Inner BA (Full rank) + struct ggml_tensor* w2a, // Inner A (Low rank part 1) + struct ggml_tensor* w2b, // Inner B (Low rank part 2) + bool is_conv, + WeightAdapter::ForwardParams::conv2d_params_t conv_params, + float scale) { + GGML_ASSERT((w1 != NULL || (w1a != NULL && w1b != NULL))); + GGML_ASSERT((w2 != NULL || (w2a != NULL && w2b != NULL))); + + int uq = (w1 != NULL) ? (int)w1->ne[0] : (int)w1a->ne[0]; + int up = (w1 != NULL) ? (int)w1->ne[1] : (int)w1b->ne[1]; + + int q_actual = is_conv ? (int)h->ne[2] : (int)h->ne[0]; + int vq = q_actual / uq; + + int vp = (w2 != NULL) ? (is_conv ? (int)w2->ne[3] : (int)w2->ne[1]) + : (int)w2a->ne[1]; + GGML_ASSERT(q_actual == (uq * vq) && "Input dimension mismatch for LoKR split"); + + struct ggml_tensor* hb; + + if (!is_conv) { + int batch = (int)h->ne[1]; + int merge_batch_uq = batch; + int merge_batch_vp = batch; + +#if SD_USE_VULKAN + if (batch > 1) { + // no access to backend here, worst case is slightly worse perfs for other backends when built alongside Vulkan backend + int max_batch = 65535; + int max_batch_uq = max_batch / uq; + merge_batch_uq = 1; + for (int i = max_batch_uq; i > 0; i--) { + if (batch % i == 0) { + merge_batch_uq = i; + break; + } + } + + int max_batch_vp = max_batch / vp; + merge_batch_vp = 1; + for (int i = max_batch_vp; i > 0; i--) { + if (batch % i == 0) { + merge_batch_vp = i; + break; + } + } + } +#endif + + struct ggml_tensor* h_split = ggml_reshape_3d(ctx, h, vq, uq * merge_batch_uq, batch / merge_batch_uq); + if (w2 != NULL) { + hb = ggml_mul_mat(ctx, w2, h_split); + } else { + hb = ggml_mul_mat(ctx, w2b, ggml_mul_mat(ctx, w2a, h_split)); + } + + if (batch > 1) { + hb = ggml_reshape_3d(ctx, hb, vp, uq, batch); + } + struct ggml_tensor* hb_t = ggml_cont(ctx, ggml_transpose(ctx, hb)); + hb_t = ggml_reshape_3d(ctx, hb_t, uq, vp * merge_batch_vp, batch / merge_batch_vp); + + struct ggml_tensor* hc_t; + if (w1 != NULL) { + hc_t = ggml_mul_mat(ctx, w1, hb_t); + } else { + hc_t = ggml_mul_mat(ctx, w1b, ggml_mul_mat(ctx, w1a, hb_t)); + } + + if (batch > 1) { + hc_t = ggml_reshape_3d(ctx, hc_t, up, vp, batch); + } + + struct ggml_tensor* hc = ggml_transpose(ctx, hc_t); + struct ggml_tensor* out = ggml_reshape_2d(ctx, ggml_cont(ctx, hc), up * vp, batch); + return ggml_scale(ctx, out, scale); + } else { + int batch = (int)h->ne[3]; + // 1. Reshape input: [W, H, vq*uq, batch] -> [W, H, vq, uq * batch] + struct ggml_tensor* h_split = ggml_reshape_4d(ctx, h, h->ne[0], h->ne[1], vq, uq * batch); + + if (w2 != NULL) { + hb = ggml_ext_conv_2d(ctx, h_split, w2, nullptr, + conv_params.s0, + conv_params.s1, + conv_params.p0, + conv_params.p1, + conv_params.d0, + conv_params.d1, + conv_params.direct, + conv_params.circular_x, + conv_params.circular_y, + conv_params.scale); + } else { + // swap a and b order for conv lora + struct ggml_tensor* a = w2b; + struct ggml_tensor* b = w2a; + + // unpack conv2d weights if needed + if (ggml_n_dims(a) < 4) { + int k = (int)sqrt(a->ne[0] / h_split->ne[2]); + GGML_ASSERT(k * k * h_split->ne[2] == a->ne[0]); + a = ggml_reshape_4d(ctx, a, k, k, a->ne[0] / (k * k), a->ne[1]); + } else if (a->ne[2] != h_split->ne[2]) { + int k = (int)sqrt(a->ne[2] / h_split->ne[2]); + GGML_ASSERT(k * k * h_split->ne[2] == a->ne[2]); + a = ggml_reshape_4d(ctx, a, a->ne[0] * k, a->ne[1] * k, a->ne[2] / (k * k), a->ne[3]); + } + struct ggml_tensor* ha = ggml_ext_conv_2d(ctx, h_split, a, nullptr, + conv_params.s0, + conv_params.s1, + conv_params.p0, + conv_params.p1, + conv_params.d0, + conv_params.d1, + conv_params.direct, + conv_params.circular_x, + conv_params.circular_y, + conv_params.scale); + + // not supporting lora_mid here + hb = ggml_ext_conv_2d(ctx, + ha, + b, + nullptr, + 1, + 1, + 0, + 0, + 1, + 1, + conv_params.direct, + conv_params.circular_x, + conv_params.circular_y, + conv_params.scale); + } + + // Current hb shape: [W_out, H_out, vp, uq * batch] + int w_out = (int)hb->ne[0]; + int h_out = (int)hb->ne[1]; + + // struct ggml_tensor* hb_cat = ggml_reshape_4d(ctx, hb, w_out , h_out , vp * uq, batch); + // [W_out, H_out, vp * uq, batch] + // Now left to compute (W1 kr Id) * hb_cat == (W1 kr W2) cv h + + // merge the uq groups of size vp*w_out*h_out + struct ggml_tensor* hb_merged = ggml_reshape_2d(ctx, hb, w_out * h_out * vp, uq * batch); + struct ggml_tensor* hc_t; + struct ggml_tensor* hb_merged_t = ggml_cont(ctx, ggml_transpose(ctx, hb_merged)); + if (w1 != NULL) { + // Would be great to be able to transpose w1 instead to avoid transposing both hb and hc + hc_t = ggml_mul_mat(ctx, w1, hb_merged_t); + } else { + hc_t = ggml_mul_mat(ctx, w1b, ggml_mul_mat(ctx, w1a, hb_merged_t)); + } + struct ggml_tensor* hc = ggml_transpose(ctx, hc_t); + // ungroup + struct ggml_tensor* out = ggml_reshape_4d(ctx, ggml_cont(ctx, hc), w_out, h_out, up * vp, batch); + return ggml_scale(ctx, out, scale); + } +} + #endif // __GGML_EXTEND__HPP__ diff --git a/otherarch/sdcpp/latent-preview.h b/otherarch/sdcpp/latent-preview.h index 76e17415c..85c8e0dcf 100644 --- a/otherarch/sdcpp/latent-preview.h +++ b/otherarch/sdcpp/latent-preview.h @@ -1,234 +1,234 @@ -#include -#include -#include "ggml.h" - -const float wan_21_latent_rgb_proj[16][3] = { - {0.015123f, -0.148418f, 0.479828f}, - {0.003652f, -0.010680f, -0.037142f}, - {0.212264f, 0.063033f, 0.016779f}, - {0.232999f, 0.406476f, 0.220125f}, - {-0.051864f, -0.082384f, -0.069396f}, - {0.085005f, -0.161492f, 0.010689f}, - {-0.245369f, -0.506846f, -0.117010f}, - {-0.151145f, 0.017721f, 0.007207f}, - {-0.293239f, -0.207936f, -0.421135f}, - {-0.187721f, 0.050783f, 0.177649f}, - {-0.013067f, 0.265964f, 0.166578f}, - {0.028327f, 0.109329f, 0.108642f}, - {-0.205343f, 0.043991f, 0.148914f}, - {0.014307f, -0.048647f, -0.007219f}, - {0.217150f, 0.053074f, 0.319923f}, - {0.155357f, 0.083156f, 0.064780f}}; -float wan_21_latent_rgb_bias[3] = {-0.270270f, -0.234976f, -0.456853f}; - -const float wan_22_latent_rgb_proj[48][3] = { - {0.017126f, -0.027230f, -0.019257f}, - {-0.113739f, -0.028715f, -0.022885f}, - {-0.000106f, 0.021494f, 0.004629f}, - {-0.013273f, -0.107137f, -0.033638f}, - {-0.000381f, 0.000279f, 0.025877f}, - {-0.014216f, -0.003975f, 0.040528f}, - {0.001638f, -0.000748f, 0.011022f}, - {0.029238f, -0.006697f, 0.035933f}, - {0.021641f, -0.015874f, 0.040531f}, - {-0.101984f, -0.070160f, -0.028855f}, - {0.033207f, -0.021068f, 0.002663f}, - {-0.104711f, 0.121673f, 0.102981f}, - {0.082647f, -0.004991f, 0.057237f}, - {-0.027375f, 0.031581f, 0.006868f}, - {-0.045434f, 0.029444f, 0.019287f}, - {-0.046572f, -0.012537f, 0.006675f}, - {0.074709f, 0.033690f, 0.025289f}, - {-0.008251f, -0.002745f, -0.006999f}, - {0.012685f, -0.061856f, -0.048658f}, - {0.042304f, -0.007039f, 0.000295f}, - {-0.007644f, -0.060843f, -0.033142f}, - {0.159909f, 0.045628f, 0.367541f}, - {0.095171f, 0.086438f, 0.010271f}, - {0.006812f, 0.019643f, 0.029637f}, - {0.003467f, -0.010705f, 0.014252f}, - {-0.099681f, -0.066272f, -0.006243f}, - {0.047357f, 0.037040f, 0.000185f}, - {-0.041797f, -0.089225f, -0.032257f}, - {0.008928f, 0.017028f, 0.018684f}, - {-0.042255f, 0.016045f, 0.006849f}, - {0.011268f, 0.036462f, 0.037387f}, - {0.011553f, -0.016375f, -0.048589f}, - {0.046266f, -0.027189f, 0.056979f}, - {0.009640f, -0.017576f, 0.030324f}, - {-0.045794f, -0.036083f, -0.010616f}, - {0.022418f, 0.039783f, -0.032939f}, - {-0.052714f, -0.015525f, 0.007438f}, - {0.193004f, 0.223541f, 0.264175f}, - {-0.059406f, -0.008188f, 0.022867f}, - {-0.156742f, -0.263791f, -0.007385f}, - {-0.015717f, 0.016570f, 0.033969f}, - {0.037969f, 0.109835f, 0.200449f}, - {-0.000782f, -0.009566f, -0.008058f}, - {0.010709f, 0.052960f, -0.044195f}, - {0.017271f, 0.045839f, 0.034569f}, - {0.009424f, 0.013088f, -0.001714f}, - {-0.024805f, -0.059378f, -0.033756f}, - {-0.078293f, 0.029070f, 0.026129f}}; -float wan_22_latent_rgb_bias[3] = {0.013160f, -0.096492f, -0.071323f}; - -const float flux_latent_rgb_proj[16][3] = { - {-0.041168f, 0.019917f, 0.097253f}, - {0.028096f, 0.026730f, 0.129576f}, - {0.065618f, -0.067950f, -0.014651f}, - {-0.012998f, -0.014762f, 0.081251f}, - {0.078567f, 0.059296f, -0.024687f}, - {-0.015987f, -0.003697f, 0.005012f}, - {0.033605f, 0.138999f, 0.068517f}, - {-0.024450f, -0.063567f, -0.030101f}, - {-0.040194f, -0.016710f, 0.127185f}, - {0.112681f, 0.088764f, -0.041940f}, - {-0.023498f, 0.093664f, 0.025543f}, - {0.082899f, 0.048320f, 0.007491f}, - {0.075712f, 0.074139f, 0.081965f}, - {-0.143501f, 0.018263f, -0.136138f}, - {-0.025767f, -0.082035f, -0.040023f}, - {-0.111849f, -0.055589f, -0.032361f}}; -float flux_latent_rgb_bias[3] = {0.024600f, -0.006937f, -0.008089f}; - -const float flux2_latent_rgb_proj[32][3] = { - {0.000736f, -0.008385f, -0.019710f}, - {-0.001352f, -0.016392f, 0.020693f}, - {-0.006376f, 0.002428f, 0.036736f}, - {0.039384f, 0.074167f, 0.119789f}, - {0.007464f, -0.005705f, -0.004734f}, - {-0.004086f, 0.005287f, -0.000409f}, - {-0.032835f, 0.050802f, -0.028120f}, - {-0.003158f, -0.000835f, 0.000406f}, - {-0.112840f, -0.084337f, -0.023083f}, - {0.001462f, -0.006656f, 0.000549f}, - {-0.009980f, -0.007480f, 0.009702f}, - {0.032540f, 0.000214f, -0.061388f}, - {0.011023f, 0.000694f, 0.007143f}, - {-0.001468f, -0.006723f, -0.001678f}, - {-0.005921f, -0.010320f, -0.003907f}, - {-0.028434f, 0.027584f, 0.018457f}, - {0.014349f, 0.011523f, 0.000441f}, - {0.009874f, 0.003081f, 0.001507f}, - {0.002218f, 0.005712f, 0.001563f}, - {0.053010f, -0.019844f, 0.008683f}, - {-0.002507f, 0.005384f, 0.000938f}, - {-0.002177f, -0.011366f, 0.003559f}, - {-0.000261f, 0.015121f, -0.003240f}, - {-0.003944f, -0.002083f, 0.005043f}, - {-0.009138f, 0.011336f, 0.003781f}, - {0.011429f, 0.003985f, -0.003855f}, - {0.010518f, -0.005586f, 0.010131f}, - {0.007883f, 0.002912f, -0.001473f}, - {-0.003318f, -0.003160f, 0.003684f}, - {-0.034560f, -0.008740f, 0.012996f}, - {0.000166f, 0.001079f, -0.012153f}, - {0.017772f, 0.000937f, -0.011953f}}; -float flux2_latent_rgb_bias[3] = {-0.028738f, -0.098463f, -0.107619f}; - -// This one was taken straight from -// https://github.com/Stability-AI/sd3.5/blob/8565799a3b41eb0c7ba976d18375f0f753f56402/sd3_impls.py#L288-L303 -// (MiT Licence) -const float sd3_latent_rgb_proj[16][3] = { - {-0.0645f, 0.0177f, 0.1052f}, - {0.0028f, 0.0312f, 0.0650f}, - {0.1848f, 0.0762f, 0.0360f}, - {0.0944f, 0.0360f, 0.0889f}, - {0.0897f, 0.0506f, -0.0364f}, - {-0.0020f, 0.1203f, 0.0284f}, - {0.0855f, 0.0118f, 0.0283f}, - {-0.0539f, 0.0658f, 0.1047f}, - {-0.0057f, 0.0116f, 0.0700f}, - {-0.0412f, 0.0281f, -0.0039f}, - {0.1106f, 0.1171f, 0.1220f}, - {-0.0248f, 0.0682f, -0.0481f}, - {0.0815f, 0.0846f, 0.1207f}, - {-0.0120f, -0.0055f, -0.0867f}, - {-0.0749f, -0.0634f, -0.0456f}, - {-0.1418f, -0.1457f, -0.1259f}, -}; -float sd3_latent_rgb_bias[3] = {0, 0, 0}; - -const float sdxl_latent_rgb_proj[4][3] = { - {0.258303f, 0.277640f, 0.329699f}, - {-0.299701f, 0.105446f, 0.014194f}, - {0.050522f, 0.186163f, -0.143257f}, - {-0.211938f, -0.149892f, -0.080036f}}; -float sdxl_latent_rgb_bias[3] = {0.144381f, -0.033313f, 0.007061f}; - -const float sd_latent_rgb_proj[4][3] = { - {0.337366f, 0.216344f, 0.257386f}, - {0.165636f, 0.386828f, 0.046994f}, - {-0.267803f, 0.237036f, 0.223517f}, - {-0.178022f, -0.200862f, -0.678514f}}; -float sd_latent_rgb_bias[3] = {-0.017478f, -0.055834f, -0.105825f}; - -void preview_latent_video(uint8_t* buffer, struct ggml_tensor* latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size) { - size_t buffer_head = 0; - - uint32_t latent_width = static_cast(latents->ne[0]); - uint32_t latent_height = static_cast(latents->ne[1]); - uint32_t dim = static_cast(latents->ne[ggml_n_dims(latents) - 1]); - uint32_t frames = 1; - if (ggml_n_dims(latents) == 4) { - frames = static_cast(latents->ne[2]); - } - - uint32_t rgb_width = latent_width * patch_size; - uint32_t rgb_height = latent_height * patch_size; - - uint32_t unpatched_dim = dim / (patch_size * patch_size); - - for (uint32_t k = 0; k < frames; k++) { - for (uint32_t rgb_x = 0; rgb_x < rgb_width; rgb_x++) { - for (uint32_t rgb_y = 0; rgb_y < rgb_height; rgb_y++) { - int latent_x = rgb_x / patch_size; - int latent_y = rgb_y / patch_size; - - int channel_offset = 0; - if (patch_size > 1) { - channel_offset = ((rgb_y % patch_size) * patch_size + (rgb_x % patch_size)); - } - - size_t latent_id = (latent_x * latents->nb[0] + latent_y * latents->nb[1] + k * latents->nb[2]); - - // should be incremented by 1 for each pixel - size_t pixel_id = k * rgb_width * rgb_height + rgb_y * rgb_width + rgb_x; - - float r = 0, g = 0, b = 0; - if (latent_rgb_proj != nullptr) { - for (uint32_t d = 0; d < unpatched_dim; d++) { - float value = *(float*)((char*)latents->data + latent_id + (d * patch_size * patch_size + channel_offset) * latents->nb[ggml_n_dims(latents) - 1]); - r += value * latent_rgb_proj[d][0]; - g += value * latent_rgb_proj[d][1]; - b += value * latent_rgb_proj[d][2]; - } - } else { - // interpret first 3 channels as RGB - r = *(float*)((char*)latents->data + latent_id + 0 * latents->nb[ggml_n_dims(latents) - 1]); - g = *(float*)((char*)latents->data + latent_id + 1 * latents->nb[ggml_n_dims(latents) - 1]); - b = *(float*)((char*)latents->data + latent_id + 2 * latents->nb[ggml_n_dims(latents) - 1]); - } - if (latent_rgb_bias != nullptr) { - // bias - r += latent_rgb_bias[0]; - g += latent_rgb_bias[1]; - b += latent_rgb_bias[2]; - } - // change range - r = r * .5f + .5f; - g = g * .5f + .5f; - b = b * .5f + .5f; - - // clamp rgb values to [0,1] range - r = r >= 0 ? r <= 1 ? r : 1 : 0; - g = g >= 0 ? g <= 1 ? g : 1 : 0; - b = b >= 0 ? b <= 1 ? b : 1 : 0; - - buffer[pixel_id * 3 + 0] = (uint8_t)(r * 255); - buffer[pixel_id * 3 + 1] = (uint8_t)(g * 255); - buffer[pixel_id * 3 + 2] = (uint8_t)(b * 255); - } - } - } -} +#include +#include +#include "ggml.h" + +const float wan_21_latent_rgb_proj[16][3] = { + {0.015123f, -0.148418f, 0.479828f}, + {0.003652f, -0.010680f, -0.037142f}, + {0.212264f, 0.063033f, 0.016779f}, + {0.232999f, 0.406476f, 0.220125f}, + {-0.051864f, -0.082384f, -0.069396f}, + {0.085005f, -0.161492f, 0.010689f}, + {-0.245369f, -0.506846f, -0.117010f}, + {-0.151145f, 0.017721f, 0.007207f}, + {-0.293239f, -0.207936f, -0.421135f}, + {-0.187721f, 0.050783f, 0.177649f}, + {-0.013067f, 0.265964f, 0.166578f}, + {0.028327f, 0.109329f, 0.108642f}, + {-0.205343f, 0.043991f, 0.148914f}, + {0.014307f, -0.048647f, -0.007219f}, + {0.217150f, 0.053074f, 0.319923f}, + {0.155357f, 0.083156f, 0.064780f}}; +float wan_21_latent_rgb_bias[3] = {-0.270270f, -0.234976f, -0.456853f}; + +const float wan_22_latent_rgb_proj[48][3] = { + {0.017126f, -0.027230f, -0.019257f}, + {-0.113739f, -0.028715f, -0.022885f}, + {-0.000106f, 0.021494f, 0.004629f}, + {-0.013273f, -0.107137f, -0.033638f}, + {-0.000381f, 0.000279f, 0.025877f}, + {-0.014216f, -0.003975f, 0.040528f}, + {0.001638f, -0.000748f, 0.011022f}, + {0.029238f, -0.006697f, 0.035933f}, + {0.021641f, -0.015874f, 0.040531f}, + {-0.101984f, -0.070160f, -0.028855f}, + {0.033207f, -0.021068f, 0.002663f}, + {-0.104711f, 0.121673f, 0.102981f}, + {0.082647f, -0.004991f, 0.057237f}, + {-0.027375f, 0.031581f, 0.006868f}, + {-0.045434f, 0.029444f, 0.019287f}, + {-0.046572f, -0.012537f, 0.006675f}, + {0.074709f, 0.033690f, 0.025289f}, + {-0.008251f, -0.002745f, -0.006999f}, + {0.012685f, -0.061856f, -0.048658f}, + {0.042304f, -0.007039f, 0.000295f}, + {-0.007644f, -0.060843f, -0.033142f}, + {0.159909f, 0.045628f, 0.367541f}, + {0.095171f, 0.086438f, 0.010271f}, + {0.006812f, 0.019643f, 0.029637f}, + {0.003467f, -0.010705f, 0.014252f}, + {-0.099681f, -0.066272f, -0.006243f}, + {0.047357f, 0.037040f, 0.000185f}, + {-0.041797f, -0.089225f, -0.032257f}, + {0.008928f, 0.017028f, 0.018684f}, + {-0.042255f, 0.016045f, 0.006849f}, + {0.011268f, 0.036462f, 0.037387f}, + {0.011553f, -0.016375f, -0.048589f}, + {0.046266f, -0.027189f, 0.056979f}, + {0.009640f, -0.017576f, 0.030324f}, + {-0.045794f, -0.036083f, -0.010616f}, + {0.022418f, 0.039783f, -0.032939f}, + {-0.052714f, -0.015525f, 0.007438f}, + {0.193004f, 0.223541f, 0.264175f}, + {-0.059406f, -0.008188f, 0.022867f}, + {-0.156742f, -0.263791f, -0.007385f}, + {-0.015717f, 0.016570f, 0.033969f}, + {0.037969f, 0.109835f, 0.200449f}, + {-0.000782f, -0.009566f, -0.008058f}, + {0.010709f, 0.052960f, -0.044195f}, + {0.017271f, 0.045839f, 0.034569f}, + {0.009424f, 0.013088f, -0.001714f}, + {-0.024805f, -0.059378f, -0.033756f}, + {-0.078293f, 0.029070f, 0.026129f}}; +float wan_22_latent_rgb_bias[3] = {0.013160f, -0.096492f, -0.071323f}; + +const float flux_latent_rgb_proj[16][3] = { + {-0.041168f, 0.019917f, 0.097253f}, + {0.028096f, 0.026730f, 0.129576f}, + {0.065618f, -0.067950f, -0.014651f}, + {-0.012998f, -0.014762f, 0.081251f}, + {0.078567f, 0.059296f, -0.024687f}, + {-0.015987f, -0.003697f, 0.005012f}, + {0.033605f, 0.138999f, 0.068517f}, + {-0.024450f, -0.063567f, -0.030101f}, + {-0.040194f, -0.016710f, 0.127185f}, + {0.112681f, 0.088764f, -0.041940f}, + {-0.023498f, 0.093664f, 0.025543f}, + {0.082899f, 0.048320f, 0.007491f}, + {0.075712f, 0.074139f, 0.081965f}, + {-0.143501f, 0.018263f, -0.136138f}, + {-0.025767f, -0.082035f, -0.040023f}, + {-0.111849f, -0.055589f, -0.032361f}}; +float flux_latent_rgb_bias[3] = {0.024600f, -0.006937f, -0.008089f}; + +const float flux2_latent_rgb_proj[32][3] = { + {0.000736f, -0.008385f, -0.019710f}, + {-0.001352f, -0.016392f, 0.020693f}, + {-0.006376f, 0.002428f, 0.036736f}, + {0.039384f, 0.074167f, 0.119789f}, + {0.007464f, -0.005705f, -0.004734f}, + {-0.004086f, 0.005287f, -0.000409f}, + {-0.032835f, 0.050802f, -0.028120f}, + {-0.003158f, -0.000835f, 0.000406f}, + {-0.112840f, -0.084337f, -0.023083f}, + {0.001462f, -0.006656f, 0.000549f}, + {-0.009980f, -0.007480f, 0.009702f}, + {0.032540f, 0.000214f, -0.061388f}, + {0.011023f, 0.000694f, 0.007143f}, + {-0.001468f, -0.006723f, -0.001678f}, + {-0.005921f, -0.010320f, -0.003907f}, + {-0.028434f, 0.027584f, 0.018457f}, + {0.014349f, 0.011523f, 0.000441f}, + {0.009874f, 0.003081f, 0.001507f}, + {0.002218f, 0.005712f, 0.001563f}, + {0.053010f, -0.019844f, 0.008683f}, + {-0.002507f, 0.005384f, 0.000938f}, + {-0.002177f, -0.011366f, 0.003559f}, + {-0.000261f, 0.015121f, -0.003240f}, + {-0.003944f, -0.002083f, 0.005043f}, + {-0.009138f, 0.011336f, 0.003781f}, + {0.011429f, 0.003985f, -0.003855f}, + {0.010518f, -0.005586f, 0.010131f}, + {0.007883f, 0.002912f, -0.001473f}, + {-0.003318f, -0.003160f, 0.003684f}, + {-0.034560f, -0.008740f, 0.012996f}, + {0.000166f, 0.001079f, -0.012153f}, + {0.017772f, 0.000937f, -0.011953f}}; +float flux2_latent_rgb_bias[3] = {-0.028738f, -0.098463f, -0.107619f}; + +// This one was taken straight from +// https://github.com/Stability-AI/sd3.5/blob/8565799a3b41eb0c7ba976d18375f0f753f56402/sd3_impls.py#L288-L303 +// (MiT Licence) +const float sd3_latent_rgb_proj[16][3] = { + {-0.0645f, 0.0177f, 0.1052f}, + {0.0028f, 0.0312f, 0.0650f}, + {0.1848f, 0.0762f, 0.0360f}, + {0.0944f, 0.0360f, 0.0889f}, + {0.0897f, 0.0506f, -0.0364f}, + {-0.0020f, 0.1203f, 0.0284f}, + {0.0855f, 0.0118f, 0.0283f}, + {-0.0539f, 0.0658f, 0.1047f}, + {-0.0057f, 0.0116f, 0.0700f}, + {-0.0412f, 0.0281f, -0.0039f}, + {0.1106f, 0.1171f, 0.1220f}, + {-0.0248f, 0.0682f, -0.0481f}, + {0.0815f, 0.0846f, 0.1207f}, + {-0.0120f, -0.0055f, -0.0867f}, + {-0.0749f, -0.0634f, -0.0456f}, + {-0.1418f, -0.1457f, -0.1259f}, +}; +float sd3_latent_rgb_bias[3] = {0, 0, 0}; + +const float sdxl_latent_rgb_proj[4][3] = { + {0.258303f, 0.277640f, 0.329699f}, + {-0.299701f, 0.105446f, 0.014194f}, + {0.050522f, 0.186163f, -0.143257f}, + {-0.211938f, -0.149892f, -0.080036f}}; +float sdxl_latent_rgb_bias[3] = {0.144381f, -0.033313f, 0.007061f}; + +const float sd_latent_rgb_proj[4][3] = { + {0.337366f, 0.216344f, 0.257386f}, + {0.165636f, 0.386828f, 0.046994f}, + {-0.267803f, 0.237036f, 0.223517f}, + {-0.178022f, -0.200862f, -0.678514f}}; +float sd_latent_rgb_bias[3] = {-0.017478f, -0.055834f, -0.105825f}; + +void preview_latent_video(uint8_t* buffer, struct ggml_tensor* latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size) { + size_t buffer_head = 0; + + uint32_t latent_width = static_cast(latents->ne[0]); + uint32_t latent_height = static_cast(latents->ne[1]); + uint32_t dim = static_cast(latents->ne[ggml_n_dims(latents) - 1]); + uint32_t frames = 1; + if (ggml_n_dims(latents) == 4) { + frames = static_cast(latents->ne[2]); + } + + uint32_t rgb_width = latent_width * patch_size; + uint32_t rgb_height = latent_height * patch_size; + + uint32_t unpatched_dim = dim / (patch_size * patch_size); + + for (uint32_t k = 0; k < frames; k++) { + for (uint32_t rgb_x = 0; rgb_x < rgb_width; rgb_x++) { + for (uint32_t rgb_y = 0; rgb_y < rgb_height; rgb_y++) { + int latent_x = rgb_x / patch_size; + int latent_y = rgb_y / patch_size; + + int channel_offset = 0; + if (patch_size > 1) { + channel_offset = ((rgb_y % patch_size) * patch_size + (rgb_x % patch_size)); + } + + size_t latent_id = (latent_x * latents->nb[0] + latent_y * latents->nb[1] + k * latents->nb[2]); + + // should be incremented by 1 for each pixel + size_t pixel_id = k * rgb_width * rgb_height + rgb_y * rgb_width + rgb_x; + + float r = 0, g = 0, b = 0; + if (latent_rgb_proj != nullptr) { + for (uint32_t d = 0; d < unpatched_dim; d++) { + float value = *(float*)((char*)latents->data + latent_id + (d * patch_size * patch_size + channel_offset) * latents->nb[ggml_n_dims(latents) - 1]); + r += value * latent_rgb_proj[d][0]; + g += value * latent_rgb_proj[d][1]; + b += value * latent_rgb_proj[d][2]; + } + } else { + // interpret first 3 channels as RGB + r = *(float*)((char*)latents->data + latent_id + 0 * latents->nb[ggml_n_dims(latents) - 1]); + g = *(float*)((char*)latents->data + latent_id + 1 * latents->nb[ggml_n_dims(latents) - 1]); + b = *(float*)((char*)latents->data + latent_id + 2 * latents->nb[ggml_n_dims(latents) - 1]); + } + if (latent_rgb_bias != nullptr) { + // bias + r += latent_rgb_bias[0]; + g += latent_rgb_bias[1]; + b += latent_rgb_bias[2]; + } + // change range + r = r * .5f + .5f; + g = g * .5f + .5f; + b = b * .5f + .5f; + + // clamp rgb values to [0,1] range + r = r >= 0 ? r <= 1 ? r : 1 : 0; + g = g >= 0 ? g <= 1 ? g : 1 : 0; + b = b >= 0 ? b <= 1 ? b : 1 : 0; + + buffer[pixel_id * 3 + 0] = (uint8_t)(r * 255); + buffer[pixel_id * 3 + 1] = (uint8_t)(g * 255); + buffer[pixel_id * 3 + 2] = (uint8_t)(b * 255); + } + } + } +} diff --git a/otherarch/sdcpp/llm.hpp b/otherarch/sdcpp/llm.hpp index f9958c5c6..b98c5a00d 100644 --- a/otherarch/sdcpp/llm.hpp +++ b/otherarch/sdcpp/llm.hpp @@ -19,6 +19,7 @@ #include #include "rope.hpp" #include "tokenize_util.h" +#include "vocab/vocab.h" namespace LLM { constexpr int LLM_GRAPH_SIZE = 10240; @@ -365,7 +366,7 @@ namespace LLM { if (merges_utf8_str.size() > 0) { load_from_merges(merges_utf8_str); } else { - load_from_merges(ModelLoader::load_qwen2_merges()); + load_from_merges(load_qwen2_merges()); } } }; @@ -466,7 +467,7 @@ namespace LLM { if (merges_utf8_str.size() > 0 && vocab_utf8_str.size() > 0) { load_from_merges(merges_utf8_str, vocab_utf8_str); } else { - load_from_merges(ModelLoader::load_mistral_merges(), ModelLoader::load_mistral_vocab_json()); + load_from_merges(load_mistral_merges(), load_mistral_vocab_json()); } } }; diff --git a/otherarch/sdcpp/lora.hpp b/otherarch/sdcpp/lora.hpp index b9269b714..f3f700db4 100644 --- a/otherarch/sdcpp/lora.hpp +++ b/otherarch/sdcpp/lora.hpp @@ -468,10 +468,10 @@ struct LoraModel : public GGMLRunner { return updown; } - ggml_tensor* get_weight_diff(const std::string& model_tensor_name, ggml_context* ctx, ggml_tensor* model_tensor, bool with_lora = true) { + ggml_tensor* get_weight_diff(const std::string& model_tensor_name, ggml_context* ctx, ggml_tensor* model_tensor, bool with_lora_and_lokr = true) { // lora ggml_tensor* diff = nullptr; - if (with_lora) { + if (with_lora_and_lokr) { diff = get_lora_weight_diff(model_tensor_name, ctx); } // diff @@ -483,7 +483,7 @@ struct LoraModel : public GGMLRunner { diff = get_loha_weight_diff(model_tensor_name, ctx); } // lokr - if (diff == nullptr) { + if (diff == nullptr && with_lora_and_lokr) { diff = get_lokr_weight_diff(model_tensor_name, ctx); } if (diff != nullptr) { @@ -514,6 +514,108 @@ struct LoraModel : public GGMLRunner { } else { key = model_tensor_name + "." + std::to_string(index); } + bool is_conv2d = forward_params.op_type == WeightAdapter::ForwardParams::op_type_t::OP_CONV2D; + + std::string lokr_w1_name = "lora." + key + ".lokr_w1"; + std::string lokr_w1_a_name = "lora." + key + ".lokr_w1_a"; + // if either of these is found, then we have a lokr lora + auto iter = lora_tensors.find(lokr_w1_name); + auto iter_a = lora_tensors.find(lokr_w1_a_name); + if (iter != lora_tensors.end() || iter_a != lora_tensors.end()) { + std::string lokr_w1_b_name = "lora." + key + ".lokr_w1_b"; + std::string lokr_w2_name = "lora." + key + ".lokr_w2"; + std::string lokr_w2_a_name = "lora." + key + ".lokr_w2_a"; + std::string lokr_w2_b_name = "lora." + key + ".lokr_w2_b"; + std::string alpha_name = "lora." + key + ".alpha"; + + ggml_tensor* lokr_w1 = nullptr; + ggml_tensor* lokr_w1_a = nullptr; + ggml_tensor* lokr_w1_b = nullptr; + ggml_tensor* lokr_w2 = nullptr; + ggml_tensor* lokr_w2_a = nullptr; + ggml_tensor* lokr_w2_b = nullptr; + + if (iter != lora_tensors.end()) { + lokr_w1 = iter->second; + } + iter = iter_a; + if (iter != lora_tensors.end()) { + lokr_w1_a = iter->second; + } + iter = lora_tensors.find(lokr_w1_b_name); + if (iter != lora_tensors.end()) { + lokr_w1_b = iter->second; + } + + iter = lora_tensors.find(lokr_w2_name); + if (iter != lora_tensors.end()) { + lokr_w2 = iter->second; + if (is_conv2d && lokr_w2->type != GGML_TYPE_F16) { + lokr_w2 = ggml_cast(ctx, lokr_w2, GGML_TYPE_F16); + } + } + iter = lora_tensors.find(lokr_w2_a_name); + if (iter != lora_tensors.end()) { + lokr_w2_a = iter->second; + if (is_conv2d && lokr_w2_a->type != GGML_TYPE_F16) { + lokr_w2_a = ggml_cast(ctx, lokr_w2_a, GGML_TYPE_F16); + } + } + iter = lora_tensors.find(lokr_w2_b_name); + if (iter != lora_tensors.end()) { + lokr_w2_b = iter->second; + if (is_conv2d && lokr_w2_b->type != GGML_TYPE_F16) { + lokr_w2_b = ggml_cast(ctx, lokr_w2_b, GGML_TYPE_F16); + } + } + + int rank = 1; + if (lokr_w1_b) { + rank = (int)lokr_w1_b->ne[ggml_n_dims(lokr_w1_b) - 1]; + } + if (lokr_w2_b) { + rank = (int)lokr_w2_b->ne[ggml_n_dims(lokr_w2_b) - 1]; + } + + float scale_value = 1.0f; + iter = lora_tensors.find(alpha_name); + if (iter != lora_tensors.end()) { + float alpha = ggml_ext_backend_tensor_get_f32(iter->second); + scale_value = alpha / rank; + applied_lora_tensors.insert(alpha_name); + } + + if (rank == 1) { + scale_value = 1.0f; + } + scale_value *= multiplier; + + auto curr_out_diff = ggml_ext_lokr_forward(ctx, x, lokr_w1, lokr_w1_a, lokr_w1_b, lokr_w2, lokr_w2_a, lokr_w2_b, is_conv2d, forward_params.conv2d, scale_value); + if (out_diff == nullptr) { + out_diff = curr_out_diff; + } else { + out_diff = ggml_concat(ctx, out_diff, curr_out_diff, 0); + } + + if (lokr_w1) + applied_lora_tensors.insert(lokr_w1_name); + if (lokr_w1_a) + applied_lora_tensors.insert(lokr_w1_a_name); + if (lokr_w1_b) + applied_lora_tensors.insert(lokr_w1_b_name); + if (lokr_w2) + applied_lora_tensors.insert(lokr_w2_name); + if (lokr_w2_a) + applied_lora_tensors.insert(lokr_w2_name); + if (lokr_w2_b) + applied_lora_tensors.insert(lokr_w2_b_name); + applied_lora_tensors.insert(alpha_name); + + index++; + continue; + } + + // not a lokr, normal lora path std::string lora_down_name = "lora." + key + ".lora_down"; std::string lora_up_name = "lora." + key + ".lora_up"; @@ -525,9 +627,7 @@ struct LoraModel : public GGMLRunner { ggml_tensor* lora_mid = nullptr; ggml_tensor* lora_down = nullptr; - bool is_conv2d = forward_params.op_type == WeightAdapter::ForwardParams::op_type_t::OP_CONV2D; - - auto iter = lora_tensors.find(lora_up_name); + iter = lora_tensors.find(lora_up_name); if (iter != lora_tensors.end()) { lora_up = iter->second; if (is_conv2d && lora_up->type != GGML_TYPE_F16) { @@ -741,9 +841,9 @@ public: : lora_models(lora_models) { } - ggml_tensor* patch_weight(ggml_context* ctx, ggml_tensor* weight, const std::string& weight_name, bool with_lora) { + ggml_tensor* patch_weight(ggml_context* ctx, ggml_tensor* weight, const std::string& weight_name, bool with_lora_and_lokr) { for (auto& lora_model : lora_models) { - ggml_tensor* diff = lora_model->get_weight_diff(weight_name, ctx, weight, with_lora); + ggml_tensor* diff = lora_model->get_weight_diff(weight_name, ctx, weight, with_lora_and_lokr); if (diff == nullptr) { continue; } diff --git a/otherarch/sdcpp/main.cpp b/otherarch/sdcpp/main.cpp index ab58ab5f0..f9e4928ea 100644 --- a/otherarch/sdcpp/main.cpp +++ b/otherarch/sdcpp/main.cpp @@ -394,12 +394,15 @@ bool save_results(const SDCliParams& cli_params, fs::path base_path = out_path; fs::path ext = out_path.has_extension() ? out_path.extension() : fs::path{}; - if (!ext.empty()) - base_path.replace_extension(); std::string ext_lower = ext.string(); std::transform(ext_lower.begin(), ext_lower.end(), ext_lower.begin(), ::tolower); bool is_jpg = (ext_lower == ".jpg" || ext_lower == ".jpeg" || ext_lower == ".jpe"); + if (!ext.empty()) { + if (is_jpg || ext_lower == ".png") { + base_path.replace_extension(); + } + } int output_begin_idx = cli_params.output_begin_idx; if (output_begin_idx < 0) { @@ -409,7 +412,7 @@ bool save_results(const SDCliParams& cli_params, auto write_image = [&](const fs::path& path, int idx) { const sd_image_t& img = results[idx]; if (!img.data) - return; + return false; std::string params = get_image_params(cli_params, ctx_params, gen_params, gen_params.seed + idx); int ok = 0; @@ -419,8 +422,11 @@ bool save_results(const SDCliParams& cli_params, ok = stbi_write_png(path.string().c_str(), img.width, img.height, img.channel, img.data, 0, params.c_str()); } LOG_INFO("save result image %d to '%s' (%s)", idx, path.string().c_str(), ok ? "success" : "failure"); + return ok != 0; }; + int sucessful_reults = 0; + if (std::regex_search(cli_params.output_path, format_specifier_regex)) { if (!is_jpg && ext_lower != ".png") ext = ".png"; @@ -429,9 +435,12 @@ bool save_results(const SDCliParams& cli_params, for (int i = 0; i < num_results; ++i) { fs::path img_path = format_frame_idx(pattern.string(), output_begin_idx + i); - write_image(img_path, i); + if (write_image(img_path, i)) { + sucessful_reults++; + } } - return true; + LOG_INFO("%d/%d images saved", sucessful_reults, num_results); + return sucessful_reults != 0; } if (cli_params.mode == VID_GEN && num_results > 1) { @@ -439,9 +448,13 @@ bool save_results(const SDCliParams& cli_params, ext = ".avi"; fs::path video_path = base_path; video_path += ext; - create_mjpg_avi_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps); - LOG_INFO("save result MJPG AVI video to '%s'", video_path.string().c_str()); - return true; + if (create_mjpg_avi_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps) == 0) { + LOG_INFO("save result MJPG AVI video to '%s'", video_path.string().c_str()); + return true; + } else { + LOG_ERROR("Failed to save result MPG AVI video to '%s'", video_path.string().c_str()); + return false; + } } if (!is_jpg && ext_lower != ".png") @@ -453,10 +466,12 @@ bool save_results(const SDCliParams& cli_params, img_path += "_" + std::to_string(output_begin_idx + i); } img_path += ext; - write_image(img_path, i); + if (write_image(img_path, i)) { + sucessful_reults++; + } } - - return true; + LOG_INFO("%d/%d images saved", sucessful_reults, num_results); + return sucessful_reults != 0; } int main(int argc, const char* argv[]) { diff --git a/otherarch/sdcpp/model.cpp b/otherarch/sdcpp/model.cpp index fa102456b..ef1752d1d 100644 --- a/otherarch/sdcpp/model.cpp +++ b/otherarch/sdcpp/model.cpp @@ -16,12 +16,6 @@ #include "model.h" #include "stable-diffusion.h" #include "util.h" -#ifndef KCPP_NO_BAKE_SD_VOCAB -#include "vocab.hpp" -#include "vocab_mistral.hpp" -#include "vocab_qwen.hpp" -#include "vocab_umt5.hpp" -#endif #include "ggml-alloc.h" #include "ggml-backend.h" @@ -1368,60 +1362,6 @@ void ModelLoader::set_wtype_override(ggml_type wtype, std::string tensor_type_ru } } -std::string ModelLoader::load_merges() { -#ifndef KCPP_NO_BAKE_SD_VOCAB - std::string merges_utf8_str(reinterpret_cast(merges_utf8_c_str), sizeof(merges_utf8_c_str)); - return merges_utf8_str; -#else - return sd_load_merges(); -#endif -} - -std::string ModelLoader::load_qwen2_merges() { -#ifndef KCPP_NO_BAKE_SD_VOCAB - std::string merges_utf8_str(reinterpret_cast(qwen2_merges_utf8_c_str), sizeof(qwen2_merges_utf8_c_str)); - return merges_utf8_str; -#else - return sd_load_qwen2_merges(); -#endif -} - -std::string ModelLoader::load_mistral_merges() { -#ifndef KCPP_NO_BAKE_SD_VOCAB - std::string merges_utf8_str(reinterpret_cast(mistral_merges_utf8_c_str), sizeof(mistral_merges_utf8_c_str)); - return merges_utf8_str; -#else - return sd_load_mistral_merges(); -#endif -} - -std::string ModelLoader::load_mistral_vocab_json() { -#ifndef KCPP_NO_BAKE_SD_VOCAB - std::string json_str(reinterpret_cast(mistral_vocab_json_utf8_c_str), sizeof(mistral_vocab_json_utf8_c_str)); - return json_str; -#else - return sd_load_mistral_vocab_json(); -#endif -} - -std::string ModelLoader::load_t5_tokenizer_json() { -#ifndef KCPP_NO_BAKE_SD_VOCAB - std::string json_str(reinterpret_cast(t5_tokenizer_json_str), sizeof(t5_tokenizer_json_str)); - return json_str; -#else - return sd_load_t5(); -#endif -} - -std::string ModelLoader::load_umt5_tokenizer_json() { -#ifndef KCPP_NO_BAKE_SD_VOCAB - std::string json_str(reinterpret_cast(umt5_tokenizer_json_str), sizeof(umt5_tokenizer_json_str)); - return json_str; -#else - return sd_load_umt5(); -#endif -} - bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_threads_p, bool enable_mmap) { int64_t process_time_ms = 0; std::atomic read_time_ms(0); diff --git a/otherarch/sdcpp/model.h b/otherarch/sdcpp/model.h index 1dd07130d..66b347ab8 100644 --- a/otherarch/sdcpp/model.h +++ b/otherarch/sdcpp/model.h @@ -332,13 +332,6 @@ public: bool tensor_should_be_converted(const TensorStorage& tensor_storage, ggml_type type); int64_t get_params_mem_size(ggml_backend_t backend, ggml_type type = GGML_TYPE_COUNT); ~ModelLoader() = default; - - static std::string load_merges(); - static std::string load_qwen2_merges(); - static std::string load_mistral_merges(); - static std::string load_mistral_vocab_json(); - static std::string load_t5_tokenizer_json(); - static std::string load_umt5_tokenizer_json(); }; #endif // __MODEL_H__ diff --git a/otherarch/sdcpp/sdtype_adapter.cpp b/otherarch/sdcpp/sdtype_adapter.cpp index 862e17c94..14ac4c5dc 100644 --- a/otherarch/sdcpp/sdtype_adapter.cpp +++ b/otherarch/sdcpp/sdtype_adapter.cpp @@ -14,17 +14,8 @@ #include #include -#define KCPP_NO_BAKE_SD_VOCAB - #include "model_adapter.h" - -std::string sd_load_merges(); -std::string sd_load_t5(); -std::string sd_load_umt5(); -std::string sd_load_qwen2_merges(); -std::string sd_load_mistral_merges(); -std::string sd_load_mistral_vocab_json(); - +#include "vocab/vocab.h" #include "flux.hpp" #include "stable-diffusion.cpp" #include "util.cpp" @@ -150,7 +141,7 @@ static std::string read_str_from_disk(std::string filepath) return output; } -std::string sd_load_merges() +std::string load_clip_merges() { static std::string mergesstr; // cached string if (!mergesstr.empty()) { @@ -160,7 +151,7 @@ std::string sd_load_merges() mergesstr = read_str_from_disk(filepath); return mergesstr; } -std::string sd_load_qwen2_merges() +std::string load_qwen2_merges() { static std::string qwenmergesstr; // cached string if (!qwenmergesstr.empty()) { @@ -170,7 +161,7 @@ std::string sd_load_qwen2_merges() qwenmergesstr = read_str_from_disk(filepath); return qwenmergesstr; } -std::string sd_load_mistral_merges() +std::string load_mistral_merges() { static std::string mistralmergesstr; // cached string if (!mistralmergesstr.empty()) { @@ -180,7 +171,7 @@ std::string sd_load_mistral_merges() mistralmergesstr = read_str_from_disk(filepath); return mistralmergesstr; } -std::string sd_load_mistral_vocab_json() +std::string load_mistral_vocab_json() { static std::string mistralvocabstr; // cached string if (!mistralvocabstr.empty()) { @@ -190,7 +181,7 @@ std::string sd_load_mistral_vocab_json() mistralvocabstr = read_str_from_disk(filepath); return mistralvocabstr; } -std::string sd_load_t5() +std::string load_t5_tokenizer_json() { static std::string t5str = ""; if (!t5str.empty()) { @@ -200,7 +191,7 @@ std::string sd_load_t5() t5str = read_str_from_disk(filepath); return t5str; } -std::string sd_load_umt5() +std::string load_umt5_tokenizer_json() { static std::string umt5str = ""; if (!umt5str.empty()) { @@ -846,23 +837,23 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs) extra_image_data.push_back(img2img_data); } - const int default_res_limit = 2048; // arbitrary, just to simplify the code - // avoid crashes due to bugs/limitations on certain models - // although it can be possible for a single side to exceed 1024, the total resolution of the image - // cannot exceed (832x832) for sd1/sd2 or (2048x2048) for sdxl/sd3/flux, to prevent crashing the server - int hard_megapixel_res_limit = default_res_limit; - - int img_hard_limit = default_res_limit; + // limit by image side + int img_hard_limit = 8192; // "large enough", just to simplify the code if (cfg_side_limit > 0) { - img_hard_limit = std::max(std::min(cfg_side_limit, default_res_limit), 64); + img_hard_limit = std::max(std::min(cfg_side_limit, img_hard_limit), 64); } - int img_soft_limit = default_res_limit; + // limit by image area: avoid crashes due to bugs/limitations on certain models + // a single side can be larger, but width*height are limited by img_soft_limit² + int img_soft_limit; + int hard_megapixel_res_limit = 2048; // hard area limit, no matter the config if (cfg_square_limit <= 0) { - cfg_square_limit = ((loadedsdver==SDVersion::VERSION_SD1 || loadedsdver==SDVersion::VERSION_SD2)?832:1024); //defaults to 1 megapixel soft e.g. 1024x1024 if unspecified + // default limit is model dependent: ~0.66 megapixel for SD1.5/SD2, 1 megapixel for most models + img_soft_limit = ((loadedsdver==SDVersion::VERSION_SD1 || loadedsdver==SDVersion::VERSION_SD2)?832:1024); + } else { + // force 64 <= limit <= hard_megapixel_res_limit + img_soft_limit = std::max(std::min(cfg_square_limit, hard_megapixel_res_limit), 64); } - img_soft_limit = std::max(std::min(cfg_square_limit, default_res_limit), 64); - img_soft_limit = std::min(hard_megapixel_res_limit, img_soft_limit); sd_fix_resolution(sd_params->width, sd_params->height, img_hard_limit, img_soft_limit); if (inputs.width != sd_params->width || inputs.height != sd_params->height) { diff --git a/otherarch/sdcpp/stable-diffusion.cpp b/otherarch/sdcpp/stable-diffusion.cpp index b1c1549c1..ef9e0822c 100644 --- a/otherarch/sdcpp/stable-diffusion.cpp +++ b/otherarch/sdcpp/stable-diffusion.cpp @@ -1702,7 +1702,7 @@ public: if (vae_tiling_params.enabled) { // split latent in 32x32 tiles and compute in several steps auto on_tiling = [&](ggml_tensor* in, ggml_tensor* out, bool init) { - first_stage_model->compute(n_threads, in, true, &out, nullptr); + return first_stage_model->compute(n_threads, in, true, &out, nullptr); }; silent_tiling(latents, result, get_vae_scale_factor(), 32, 0.5f, on_tiling); @@ -1721,7 +1721,7 @@ public: if (vae_tiling_params.enabled) { // split latent in 64x64 tiles and compute in several steps auto on_tiling = [&](ggml_tensor* in, ggml_tensor* out, bool init) { - tae_first_stage->compute(n_threads, in, true, &out, nullptr); + return tae_first_stage->compute(n_threads, in, true, &out, nullptr); }; silent_tiling(latents, result, get_vae_scale_factor(), 64, 0.5f, on_tiling); } else { @@ -2690,7 +2690,7 @@ public: LOG_DEBUG("VAE Tile size: %dx%d", tile_size_x, tile_size_y); auto on_tiling = [&](ggml_tensor* in, ggml_tensor* out, bool init) { - first_stage_model->compute(n_threads, in, false, &out, work_ctx); + return first_stage_model->compute(n_threads, in, false, &out, work_ctx); }; sd_tiling_non_square(x, result, vae_scale_factor, tile_size_x, tile_size_y, tile_overlap, on_tiling); } else { @@ -2701,7 +2701,7 @@ public: if (vae_tiling_params.enabled && !encode_video) { // split latent in 32x32 tiles and compute in several steps auto on_tiling = [&](ggml_tensor* in, ggml_tensor* out, bool init) { - tae_first_stage->compute(n_threads, in, false, &out, nullptr); + return tae_first_stage->compute(n_threads, in, false, &out, nullptr); }; sd_tiling(x, result, vae_scale_factor, 64, 0.5f, on_tiling); } else { @@ -2819,11 +2819,15 @@ public: // split latent in 32x32 tiles and compute in several steps auto on_tiling = [&](ggml_tensor* in, ggml_tensor* out, bool init) { - first_stage_model->compute(n_threads, in, true, &out, nullptr); + return first_stage_model->compute(n_threads, in, true, &out, nullptr); }; sd_tiling_non_square(x, result, vae_scale_factor, tile_size_x, tile_size_y, tile_overlap, on_tiling); } else { - first_stage_model->compute(n_threads, x, true, &result, work_ctx); + if (!first_stage_model->compute(n_threads, x, true, &result, work_ctx)) { + LOG_ERROR("Failed to decode latetnts"); + first_stage_model->free_compute_buffer(); + return nullptr; + } } first_stage_model->free_compute_buffer(); process_vae_output_tensor(result); @@ -2831,11 +2835,15 @@ public: if (vae_tiling_params.enabled) { // split latent in 64x64 tiles and compute in several steps auto on_tiling = [&](ggml_tensor* in, ggml_tensor* out, bool init) { - tae_first_stage->compute(n_threads, in, true, &out); + return tae_first_stage->compute(n_threads, in, true, &out); }; sd_tiling(x, result, vae_scale_factor, 64, 0.5f, on_tiling); } else { - tae_first_stage->compute(n_threads, x, true, &result); + if (!tae_first_stage->compute(n_threads, x, true, &result)) { + LOG_ERROR("Failed to decode latetnts"); + tae_first_stage->free_compute_buffer(); + return nullptr; + } } tae_first_stage->free_compute_buffer(); } @@ -3625,6 +3633,7 @@ sd_image_t* generate_image_internal(sd_ctx_t* sd_ctx, ggml_free(work_ctx); return nullptr; } + memset(result_images, 0, batch_count * sizeof(sd_image_t)); for (size_t i = 0; i < decoded_images.size(); i++) { result_images[i].width = width; diff --git a/otherarch/sdcpp/t5.hpp b/otherarch/sdcpp/t5.hpp index 54ba00afa..b465c3427 100644 --- a/otherarch/sdcpp/t5.hpp +++ b/otherarch/sdcpp/t5.hpp @@ -14,6 +14,7 @@ #include "ggml_extend.hpp" #include #include "model.h" +#include "vocab/vocab.h" // Port from: https://github.com/google/sentencepiece/blob/master/src/unigram_model.h // and https://github.com/google/sentencepiece/blob/master/src/unigram_model.h. @@ -341,9 +342,9 @@ protected: public: explicit T5UniGramTokenizer(bool is_umt5 = false) { if (is_umt5) { - InitializePieces(ModelLoader::load_umt5_tokenizer_json()); + InitializePieces(load_umt5_tokenizer_json()); } else { - InitializePieces(ModelLoader::load_t5_tokenizer_json()); + InitializePieces(load_t5_tokenizer_json()); } min_score_ = FLT_MAX; diff --git a/otherarch/sdcpp/upscaler.cpp b/otherarch/sdcpp/upscaler.cpp index 29ac981e6..fd0dc8242 100644 --- a/otherarch/sdcpp/upscaler.cpp +++ b/otherarch/sdcpp/upscaler.cpp @@ -89,7 +89,7 @@ struct UpscalerGGML { ggml_tensor* upscaled = ggml_new_tensor_4d(upscale_ctx, GGML_TYPE_F32, output_width, output_height, 3, 1); auto on_tiling = [&](ggml_tensor* in, ggml_tensor* out, bool init) { - esrgan_upscaler->compute(n_threads, in, &out); + return esrgan_upscaler->compute(n_threads, in, &out); }; int64_t t0 = ggml_time_ms(); sd_tiling(input_image_tensor, upscaled, esrgan_upscaler->scale, esrgan_upscaler->tile_size, 0.25f, on_tiling); diff --git a/otherarch/sdcpp/vocab.hpp b/otherarch/sdcpp/vocab/clip_t5.hpp similarity index 99% rename from otherarch/sdcpp/vocab.hpp rename to otherarch/sdcpp/vocab/clip_t5.hpp index 3902045e6..8ba407836 100644 --- a/otherarch/sdcpp/vocab.hpp +++ b/otherarch/sdcpp/vocab/clip_t5.hpp @@ -1,4 +1,4 @@ -static unsigned char merges_utf8_c_str[] = { +static const unsigned char clip_merges_utf8_c_str[] = { 0x23, 0x76, 0x65, @@ -524620,7 +524620,7 @@ static unsigned char merges_utf8_c_str[] = { 0x0a, }; -static unsigned char t5_tokenizer_json_str[] = { +static const unsigned char t5_tokenizer_json_str[] = { 0x7b, 0x0a, 0x20, diff --git a/otherarch/sdcpp/vocab_mistral.hpp b/otherarch/sdcpp/vocab/mistral.hpp similarity index 99% rename from otherarch/sdcpp/vocab_mistral.hpp rename to otherarch/sdcpp/vocab/mistral.hpp index 3eb8b2591..5bfa873bc 100644 --- a/otherarch/sdcpp/vocab_mistral.hpp +++ b/otherarch/sdcpp/vocab/mistral.hpp @@ -1,4 +1,4 @@ -unsigned char mistral_merges_utf8_c_str[] = { +static const unsigned char mistral_merges_utf8_c_str[] = { 0xc4, 0xa0, 0x20, 0xc4, 0xa0, 0x0a, 0xc4, 0xa0, 0x20, 0x74, 0x0a, 0x65, 0x20, 0x72, 0x0a, 0x69, 0x20, 0x6e, 0x0a, 0xc4, 0xa0, 0x20, 0xc4, 0xa0, 0xc4, 0xa0, 0xc4, 0xa0, 0x0a, 0xc4, 0xa0, 0xc4, 0xa0, 0x20, 0xc4, 0xa0, @@ -260614,7 +260614,7 @@ unsigned char mistral_merges_utf8_c_str[] = { 0xc3, 0xa5, 0xc4, 0xb2, 0xc4, 0xb0, 0x20, 0xc3, 0xa6, 0xc2, 0xb1, 0xc4, 0xab, 0xc3, 0xa4, 0xc2, 0xb9, 0xc2, 0xa6, 0x0a, }; -unsigned char mistral_vocab_json_utf8_c_str[] = { +static const unsigned char mistral_vocab_json_utf8_c_str[] = { 0x7b, 0x22, 0x3c, 0x75, 0x6e, 0x6b, 0x3e, 0x22, 0x3a, 0x20, 0x30, 0x2c, 0x20, 0x22, 0x3c, 0x73, 0x3e, 0x22, 0x3a, 0x20, 0x31, 0x2c, 0x20, 0x22, 0x3c, 0x2f, 0x73, 0x3e, 0x22, 0x3a, 0x20, 0x32, 0x2c, 0x20, 0x22, 0x5b, diff --git a/otherarch/sdcpp/vocab_qwen.hpp b/otherarch/sdcpp/vocab/qwen.hpp similarity index 99% rename from otherarch/sdcpp/vocab_qwen.hpp rename to otherarch/sdcpp/vocab/qwen.hpp index 2c5c7fe88..9db2339e6 100644 --- a/otherarch/sdcpp/vocab_qwen.hpp +++ b/otherarch/sdcpp/vocab/qwen.hpp @@ -1,4 +1,4 @@ -unsigned char qwen2_merges_utf8_c_str[] = { +static const unsigned char qwen2_merges_utf8_c_str[] = { 0xc4, 0xa0, 0x20, 0xc4, 0xa0, 0x0a, 0xc4, 0xa0, 0xc4, 0xa0, 0x20, 0xc4, 0xa0, 0xc4, 0xa0, 0x0a, 0x69, 0x20, 0x6e, 0x0a, 0xc4, 0xa0, 0x20, 0x74, 0x0a, 0xc4, 0xa0, 0xc4, 0xa0, 0xc4, 0xa0, 0xc4, 0xa0, 0x20, 0xc4, 0xa0, diff --git a/otherarch/sdcpp/vocab_umt5.hpp b/otherarch/sdcpp/vocab/umt5.hpp similarity index 99% rename from otherarch/sdcpp/vocab_umt5.hpp rename to otherarch/sdcpp/vocab/umt5.hpp index 22c581d51..a9f87a20c 100644 --- a/otherarch/sdcpp/vocab_umt5.hpp +++ b/otherarch/sdcpp/vocab/umt5.hpp @@ -1,4 +1,4 @@ -unsigned char umt5_tokenizer_json_str[] = { +static const unsigned char umt5_tokenizer_json_str[] = { 0x7b, 0x22, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x3a, 0x20, 0x22, 0x31, 0x2e, 0x30, 0x22, 0x2c, 0x20, 0x22, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3a, 0x20, 0x6e, 0x75, 0x6c, diff --git a/otherarch/sdcpp/vocab/vocab.cpp b/otherarch/sdcpp/vocab/vocab.cpp new file mode 100644 index 000000000..63b286860 --- /dev/null +++ b/otherarch/sdcpp/vocab/vocab.cpp @@ -0,0 +1,35 @@ +#include "vocab.h" +#include "clip_t5.hpp" +#include "mistral.hpp" +#include "qwen.hpp" +#include "umt5.hpp" + +std::string load_clip_merges() { + std::string merges_utf8_str(reinterpret_cast(clip_merges_utf8_c_str), sizeof(clip_merges_utf8_c_str)); + return merges_utf8_str; +} + +std::string load_qwen2_merges() { + std::string merges_utf8_str(reinterpret_cast(qwen2_merges_utf8_c_str), sizeof(qwen2_merges_utf8_c_str)); + return merges_utf8_str; +} + +std::string load_mistral_merges() { + std::string merges_utf8_str(reinterpret_cast(mistral_merges_utf8_c_str), sizeof(mistral_merges_utf8_c_str)); + return merges_utf8_str; +} + +std::string load_mistral_vocab_json() { + std::string json_str(reinterpret_cast(mistral_vocab_json_utf8_c_str), sizeof(mistral_vocab_json_utf8_c_str)); + return json_str; +} + +std::string load_t5_tokenizer_json() { + std::string json_str(reinterpret_cast(t5_tokenizer_json_str), sizeof(t5_tokenizer_json_str)); + return json_str; +} + +std::string load_umt5_tokenizer_json() { + std::string json_str(reinterpret_cast(umt5_tokenizer_json_str), sizeof(umt5_tokenizer_json_str)); + return json_str; +} \ No newline at end of file diff --git a/otherarch/sdcpp/vocab/vocab.h b/otherarch/sdcpp/vocab/vocab.h new file mode 100644 index 000000000..cfa033a49 --- /dev/null +++ b/otherarch/sdcpp/vocab/vocab.h @@ -0,0 +1,13 @@ +#ifndef __VOCAB_H__ +#define __VOCAB_H__ + +#include + +std::string load_clip_merges(); +std::string load_qwen2_merges(); +std::string load_mistral_merges(); +std::string load_mistral_vocab_json(); +std::string load_t5_tokenizer_json(); +std::string load_umt5_tokenizer_json(); + +#endif // __VOCAB_H__ \ No newline at end of file