From 8ef66e90c140dfb2ed4a90b35143fd1d58a525cb Mon Sep 17 00:00:00 2001 From: Wagner Bruna Date: Sun, 16 Nov 2025 07:16:28 -0300 Subject: [PATCH] sd: sync to master-366-f532972 --- otherarch/sdcpp/common.hpp | 8 +- otherarch/sdcpp/conditioner.hpp | 39 ++++++- otherarch/sdcpp/main.cpp | 63 ++++++++---- otherarch/sdcpp/model.cpp | 87 +++++++++------- otherarch/sdcpp/model.h | 2 +- otherarch/sdcpp/qwen_image.hpp | 8 +- otherarch/sdcpp/rng_mt19937.hpp | 147 +++++++++++++++++++++++++++ otherarch/sdcpp/sdtype_adapter.cpp | 5 - otherarch/sdcpp/stable-diffusion.cpp | 93 +++++++++++------ otherarch/sdcpp/stable-diffusion.h | 3 + otherarch/sdcpp/util.cpp | 9 +- 11 files changed, 359 insertions(+), 105 deletions(-) create mode 100644 otherarch/sdcpp/rng_mt19937.hpp diff --git a/otherarch/sdcpp/common.hpp b/otherarch/sdcpp/common.hpp index dd8281f9e..33d499fb1 100644 --- a/otherarch/sdcpp/common.hpp +++ b/otherarch/sdcpp/common.hpp @@ -242,14 +242,18 @@ public: } // net_1 is nn.Dropout(), skip for inference - float scale = 1.f; + bool force_prec_f32 = false; + float scale = 1.f; if (precision_fix) { scale = 1.f / 128.f; +#ifdef SD_USE_VULKAN + force_prec_f32 = true; +#endif } // The purpose of the scale here is to prevent NaN issues in certain situations. // For example, when using Vulkan without enabling force_prec_f32, // or when using CUDA but the weights are k-quants. - blocks["net.2"] = std::shared_ptr(new Linear(inner_dim, dim_out, true, false, false, scale)); + blocks["net.2"] = std::shared_ptr(new Linear(inner_dim, dim_out, true, false, force_prec_f32, scale)); } struct ggml_tensor* forward(GGMLRunnerContext* ctx, struct ggml_tensor* x) { diff --git a/otherarch/sdcpp/conditioner.hpp b/otherarch/sdcpp/conditioner.hpp index 27d367a9c..94e98a511 100644 --- a/otherarch/sdcpp/conditioner.hpp +++ b/otherarch/sdcpp/conditioner.hpp @@ -278,13 +278,30 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { const std::string& curr_text = item.first; float curr_weight = item.second; // printf(" %s: %f \n", curr_text.c_str(), curr_weight); + int32_t clean_index = 0; + if (curr_text == "BREAK" && curr_weight == -1.0f) { + // Pad token array up to chunk size at this point. + // TODO: This is a hardcoded chunk_len, like in stable-diffusion.cpp, make it a parameter for the future? + // Also, this is 75 instead of 77 to leave room for BOS and EOS tokens. + int padding_size = 75 - (tokens_acc % 75); + for (int j = 0; j < padding_size; j++) { + clean_input_ids.push_back(tokenizer.EOS_TOKEN_ID); + clean_index++; + } + + // After padding, continue to the next iteration to process the following text as a new segment + tokens.insert(tokens.end(), clean_input_ids.begin(), clean_input_ids.end()); + weights.insert(weights.end(), padding_size, curr_weight); + continue; + } + + // Regular token, process normally std::vector curr_tokens = tokenizer.encode(curr_text, on_new_token_cb); - int32_t clean_index = 0; for (uint32_t i = 0; i < curr_tokens.size(); i++) { int token_id = curr_tokens[i]; - if (token_id == image_token) + if (token_id == image_token) { class_token_index.push_back(clean_index - 1); - else { + } else { clean_input_ids.push_back(token_id); clean_index++; } @@ -387,6 +404,22 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { for (const auto& item : parsed_attention) { const std::string& curr_text = item.first; float curr_weight = item.second; + + if (curr_text == "BREAK" && curr_weight == -1.0f) { + // Pad token array up to chunk size at this point. + // TODO: This is a hardcoded chunk_len, like in stable-diffusion.cpp, make it a parameter for the future? + // Also, this is 75 instead of 77 to leave room for BOS and EOS tokens. + size_t current_size = tokens.size(); + size_t padding_size = (75 - (current_size % 75)) % 75; // Ensure no negative padding + + if (padding_size > 0) { + LOG_DEBUG("BREAK token encountered, padding current chunk by %zu tokens.", padding_size); + tokens.insert(tokens.end(), padding_size, tokenizer.EOS_TOKEN_ID); + weights.insert(weights.end(), padding_size, 1.0f); + } + continue; // Skip to the next item after handling BREAK + } + std::vector curr_tokens = tokenizer.encode(curr_text, on_new_token_cb); tokens.insert(tokens.end(), curr_tokens.begin(), curr_tokens.end()); weights.insert(weights.end(), curr_tokens.size(), curr_weight); diff --git a/otherarch/sdcpp/main.cpp b/otherarch/sdcpp/main.cpp index 80612979c..a6c82086b 100644 --- a/otherarch/sdcpp/main.cpp +++ b/otherarch/sdcpp/main.cpp @@ -110,21 +110,22 @@ struct SDParams { int fps = 16; float vace_strength = 1.f; - float strength = 0.75f; - float control_strength = 0.9f; - rng_type_t rng_type = CUDA_RNG; - int64_t seed = 42; - bool verbose = false; - bool offload_params_to_cpu = false; - bool control_net_cpu = false; - bool clip_on_cpu = false; - bool vae_on_cpu = false; - bool diffusion_flash_attn = false; - bool diffusion_conv_direct = false; - bool vae_conv_direct = false; - bool canny_preprocess = false; - bool color = false; - int upscale_repeats = 1; + float strength = 0.75f; + float control_strength = 0.9f; + rng_type_t rng_type = CUDA_RNG; + rng_type_t sampler_rng_type = RNG_TYPE_COUNT; + int64_t seed = 42; + bool verbose = false; + bool offload_params_to_cpu = false; + bool control_net_cpu = false; + bool clip_on_cpu = false; + bool vae_on_cpu = false; + bool diffusion_flash_attn = false; + bool diffusion_conv_direct = false; + bool vae_conv_direct = false; + bool canny_preprocess = false; + bool color = false; + int upscale_repeats = 1; // Photo Maker std::string photo_maker_path; @@ -214,6 +215,7 @@ void print_params(SDParams params) { printf(" flow_shift: %.2f\n", params.flow_shift); printf(" strength(img2img): %.2f\n", params.strength); printf(" rng: %s\n", sd_rng_type_name(params.rng_type)); + printf(" sampler rng: %s\n", sd_rng_type_name(params.sampler_rng_type)); printf(" seed: %zd\n", params.seed); printf(" batch_count: %d\n", params.batch_count); printf(" vae_tiling: %s\n", params.vae_tiling_params.enabled ? "true" : "false"); @@ -886,6 +888,20 @@ void parse_args(int argc, const char** argv, SDParams& params) { return 1; }; + auto on_sampler_rng_arg = [&](int argc, const char** argv, int index) { + if (++index >= argc) { + return -1; + } + const char* arg = argv[index]; + params.sampler_rng_type = str_to_rng_type(arg); + if (params.sampler_rng_type == RNG_TYPE_COUNT) { + fprintf(stderr, "error: invalid sampler rng type %s\n", + arg); + return -1; + } + return 1; + }; + auto on_schedule_arg = [&](int argc, const char** argv, int index) { if (++index >= argc) { return -1; @@ -1124,8 +1140,12 @@ void parse_args(int argc, const char** argv, SDParams& params) { on_type_arg}, {"", "--rng", - "RNG, one of [std_default, cuda], default: cuda", + "RNG, one of [std_default, cuda, cpu], default: cuda(sd-webui), cpu(comfyui)", on_rng_arg}, + {"", + "--sampler-rng", + "sampler RNG, one of [std_default, cuda, cpu]. If not specified, use --rng", + on_sampler_rng_arg}, {"-s", "--seed", "RNG seed (default: 42, use random seed for < 0)", @@ -1144,7 +1164,7 @@ void parse_args(int argc, const char** argv, SDParams& params) { "the way to apply LoRA, one of [auto, immediately, at_runtime], default is auto. " "In auto mode, if the model weights contain any quantized parameters, the at_runtime mode will be used; otherwise, immediately will be used." "The immediately mode may have precision and compatibility issues with quantized parameters, " - "but it usually offers faster inference speed and, in some cases, lower memory usage" + "but it usually offers faster inference speed and, in some cases, lower memory usage. " "The at_runtime mode, on the other hand, is exactly the opposite.", on_lora_apply_mode_arg}, {"", @@ -1241,10 +1261,6 @@ void parse_args(int argc, const char** argv, SDParams& params) { exit(1); } - if (params.mode != CONVERT && params.tensor_type_rules.size() > 0) { - fprintf(stderr, "warning: --tensor-type-rules is currently supported only for conversion\n"); - } - if (params.mode == VID_GEN && params.video_frames <= 0) { fprintf(stderr, "warning: --video-frames must be at least 1\n"); exit(1); @@ -1323,6 +1339,9 @@ std::string get_image_params(SDParams params, int64_t seed) { parameter_string += "Size: " + std::to_string(params.width) + "x" + std::to_string(params.height) + ", "; parameter_string += "Model: " + sd_basename(params.model_path) + ", "; parameter_string += "RNG: " + std::string(sd_rng_type_name(params.rng_type)) + ", "; + if (params.sampler_rng_type != RNG_TYPE_COUNT) { + parameter_string += "Sampler RNG: " + std::string(sd_rng_type_name(params.sampler_rng_type)) + ", "; + } parameter_string += "Sampler: " + std::string(sd_sample_method_name(params.sample_params.sample_method)); if (params.sample_params.scheduler != DEFAULT) { parameter_string += " " + std::string(sd_schedule_name(params.sample_params.scheduler)); @@ -1756,11 +1775,13 @@ int main(int argc, const char* argv[]) { params.lora_model_dir.c_str(), params.embedding_dir.c_str(), params.photo_maker_path.c_str(), + params.tensor_type_rules.c_str(), vae_decode_only, true, params.n_threads, params.wtype, params.rng_type, + params.sampler_rng_type, params.prediction, params.lora_apply_mode, params.offload_params_to_cpu, diff --git a/otherarch/sdcpp/model.cpp b/otherarch/sdcpp/model.cpp index dc8fea17d..f0ff3f2cd 100644 --- a/otherarch/sdcpp/model.cpp +++ b/otherarch/sdcpp/model.cpp @@ -1312,15 +1312,59 @@ std::map ModelLoader::get_vae_wtype_stat() { return wtype_stat; } -void ModelLoader::set_wtype_override(ggml_type wtype, std::string prefix) { +static std::vector> parse_tensor_type_rules(const std::string& tensor_type_rules) { + std::vector> result; + for (const auto& item : split_string(tensor_type_rules, ',')) { + if (item.size() == 0) + continue; + std::string::size_type pos = item.find('='); + if (pos == std::string::npos) { + LOG_WARN("ignoring invalid quant override \"%s\"", item.c_str()); + continue; + } + std::string tensor_pattern = item.substr(0, pos); + std::string type_name = item.substr(pos + 1); + + ggml_type tensor_type = GGML_TYPE_COUNT; + + if (type_name == "f32") { + tensor_type = GGML_TYPE_F32; + } else { + for (size_t i = 0; i < GGML_TYPE_COUNT; i++) { + auto trait = ggml_get_type_traits((ggml_type)i); + if (trait->to_float && trait->type_size && type_name == trait->type_name) { + tensor_type = (ggml_type)i; + } + } + } + + if (tensor_type != GGML_TYPE_COUNT) { + result.emplace_back(tensor_pattern, tensor_type); + } else { + LOG_WARN("ignoring invalid quant override \"%s\"", item.c_str()); + } + } + return result; +} + +void ModelLoader::set_wtype_override(ggml_type wtype, std::string tensor_type_rules) { + auto map_rules = parse_tensor_type_rules(tensor_type_rules); for (auto& [name, tensor_storage] : tensor_storage_map) { - if (!starts_with(name, prefix)) { + ggml_type dst_type = wtype; + for (const auto& tensor_type_rule : map_rules) { + std::regex pattern(tensor_type_rule.first); + if (std::regex_search(name, pattern)) { + dst_type = tensor_type_rule.second; + break; + } + } + if (dst_type == GGML_TYPE_COUNT) { continue; } - if (!tensor_should_be_converted(tensor_storage, wtype)) { + if (!tensor_should_be_converted(tensor_storage, dst_type)) { continue; } - tensor_storage.expected_type = wtype; + tensor_storage.expected_type = dst_type; } } @@ -1683,41 +1727,6 @@ bool ModelLoader::load_tensors(std::map& tenso return true; } -std::vector> parse_tensor_type_rules(const std::string& tensor_type_rules) { - std::vector> result; - for (const auto& item : split_string(tensor_type_rules, ',')) { - if (item.size() == 0) - continue; - std::string::size_type pos = item.find('='); - if (pos == std::string::npos) { - LOG_WARN("ignoring invalid quant override \"%s\"", item.c_str()); - continue; - } - std::string tensor_pattern = item.substr(0, pos); - std::string type_name = item.substr(pos + 1); - - ggml_type tensor_type = GGML_TYPE_COUNT; - - if (type_name == "f32") { - tensor_type = GGML_TYPE_F32; - } else { - for (size_t i = 0; i < GGML_TYPE_COUNT; i++) { - auto trait = ggml_get_type_traits((ggml_type)i); - if (trait->to_float && trait->type_size && type_name == trait->type_name) { - tensor_type = (ggml_type)i; - } - } - } - - if (tensor_type != GGML_TYPE_COUNT) { - result.emplace_back(tensor_pattern, tensor_type); - } else { - LOG_WARN("ignoring invalid quant override \"%s\"", item.c_str()); - } - } - return result; -} - bool ModelLoader::tensor_should_be_converted(const TensorStorage& tensor_storage, ggml_type type) { const std::string& name = tensor_storage.name; if (type != GGML_TYPE_COUNT) { diff --git a/otherarch/sdcpp/model.h b/otherarch/sdcpp/model.h index 1c8066074..e2b607ce7 100644 --- a/otherarch/sdcpp/model.h +++ b/otherarch/sdcpp/model.h @@ -293,7 +293,7 @@ public: std::map get_diffusion_model_wtype_stat(); std::map get_vae_wtype_stat(); String2TensorStorage& get_tensor_storage_map() { return tensor_storage_map; } - void set_wtype_override(ggml_type wtype, std::string prefix = ""); + void set_wtype_override(ggml_type wtype, std::string tensor_type_rules = ""); bool load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_threads = 0); bool load_tensors(std::map& tensors, std::set ignore_tensors = {}, diff --git a/otherarch/sdcpp/qwen_image.hpp b/otherarch/sdcpp/qwen_image.hpp index 94ada47d7..3e4a75e07 100644 --- a/otherarch/sdcpp/qwen_image.hpp +++ b/otherarch/sdcpp/qwen_image.hpp @@ -94,10 +94,14 @@ namespace Qwen { blocks["norm_added_q"] = std::shared_ptr(new RMSNorm(dim_head, eps)); blocks["norm_added_k"] = std::shared_ptr(new RMSNorm(dim_head, eps)); - float scale = 1.f / 32.f; + float scale = 1.f / 32.f; + bool force_prec_f32 = false; +#ifdef SD_USE_VULKAN + force_prec_f32 = true; +#endif // The purpose of the scale here is to prevent NaN issues in certain situations. // For example when using CUDA but the weights are k-quants (not all prompts). - blocks["to_out.0"] = std::shared_ptr(new Linear(inner_dim, out_dim, out_bias, false, false, scale)); + blocks["to_out.0"] = std::shared_ptr(new Linear(inner_dim, out_dim, out_bias, false, force_prec_f32, scale)); // to_out.1 is nn.Dropout blocks["to_add_out"] = std::shared_ptr(new Linear(inner_dim, out_context_dim, out_bias, false, false, scale)); diff --git a/otherarch/sdcpp/rng_mt19937.hpp b/otherarch/sdcpp/rng_mt19937.hpp new file mode 100644 index 000000000..7e6199886 --- /dev/null +++ b/otherarch/sdcpp/rng_mt19937.hpp @@ -0,0 +1,147 @@ +#ifndef __RNG_MT19937_HPP__ +#define __RNG_MT19937_HPP__ + +#include +#include + +#include "rng.hpp" + +// RNG imitiating torch cpu randn on CPU. +// Port from pytorch, original license: https://github.com/pytorch/pytorch/blob/d01a7b0241ed1c4cded7e7ca097249feb343f072/LICENSE +// Ref: https://github.com/pytorch/pytorch/blob/d01a7b0241ed1c4cded7e7ca097249feb343f072/aten/src/ATen/core/TransformationHelper.h, for uniform_real +// Ref: https://github.com/pytorch/pytorch/blob/d01a7b0241ed1c4cded7e7ca097249feb343f072/aten/src/ATen/native/cpu/DistributionTemplates.h, for normal_kernel/normal_fill/normal_fill_16 +// Ref: https://github.com/pytorch/pytorch/blob/d01a7b0241ed1c4cded7e7ca097249feb343f072/aten/src/ATen/core/MT19937RNGEngine.h, for mt19937_engine +// Ref: https://github.com/pytorch/pytorch/blob/d01a7b0241ed1c4cded7e7ca097249feb343f072/aten/src/ATen/core/DistributionsHelper.h, for uniform_real_distribution/normal_distribution +class MT19937RNG : public RNG { + static const int N = 624; + static const int M = 397; + static const uint32_t MATRIX_A = 0x9908b0dfU; + static const uint32_t UMASK = 0x80000000U; + static const uint32_t LMASK = 0x7fffffffU; + + struct State { + uint64_t seed_; + int left_; + bool seeded_; + uint32_t next_; + std::array state_; + bool has_next_gauss = false; + double next_gauss = 0.0f; + }; + + State s; + + uint32_t mix_bits(uint32_t u, uint32_t v) { return (u & UMASK) | (v & LMASK); } + uint32_t twist(uint32_t u, uint32_t v) { return (mix_bits(u, v) >> 1) ^ ((v & 1) ? MATRIX_A : 0); } + void next_state() { + uint32_t* p = s.state_.data(); + s.left_ = N; + s.next_ = 0; + for (int j = N - M + 1; --j; p++) + p[0] = p[M] ^ twist(p[0], p[1]); + for (int j = M; --j; p++) + p[0] = p[M - N] ^ twist(p[0], p[1]); + p[0] = p[M - N] ^ twist(p[0], s.state_[0]); + } + + uint32_t rand_uint32() { + if (--s.left_ == 0) + next_state(); + uint32_t y = s.state_[s.next_++]; + y ^= (y >> 11); + y ^= (y << 7) & 0x9d2c5680U; + y ^= (y << 15) & 0xefc60000U; + y ^= (y >> 18); + return y; + } + + uint64_t rand_uint64() { + uint64_t high = (uint64_t)rand_uint32(); + uint64_t low = (uint64_t)rand_uint32(); + return (high << 32) | low; + } + + template + T uniform_real(V val, T from, T to) { + constexpr auto MASK = static_cast((static_cast(1) << std::numeric_limits::digits) - 1); + constexpr auto DIVISOR = static_cast(1) / (static_cast(1) << std::numeric_limits::digits); + T x = (val & MASK) * DIVISOR; + return (x * (to - from) + from); + } + + double normal_double_value(double mean, double std) { + if (s.has_next_gauss) { + s.has_next_gauss = false; + return s.next_gauss; + } + double u1 = uniform_real(rand_uint64(), 0., 1.); // double + double u2 = uniform_real(rand_uint64(), 0., 1.); // double + + double r = std::sqrt(-2.0 * std::log1p(-u2)); + double theta = 2.0 * 3.14159265358979323846 * u1; + double value = r * std::cos(theta) * std + mean; + s.next_gauss = r * std::sin(theta) * std + mean; + s.has_next_gauss = true; + return value; + } + + void normal_fill_16(float* data, float mean, float std) { + for (int j = 0; j < 8; ++j) { + float u1 = 1.0f - data[j]; + float u2 = data[j + 8]; + float r = std::sqrt(-2.0f * std::log(u1)); + float theta = 2.0f * 3.14159265358979323846 * u2; + data[j] = r * std::cos(theta) * std + mean; + data[j + 8] = r * std::sin(theta) * std + mean; + } + } + + void randn(float* data, int64_t size, float mean = 0.0f, float std = 1.0f) { + if (size >= 16) { + for (int64_t i = 0; i < size; i++) { + data[i] = uniform_real(rand_uint32(), 0.f, 1.f); + } + for (int64_t i = 0; i < size - 15; i += 16) { + normal_fill_16(data + i, mean, std); + } + if (size % 16 != 0) { + // Recompute the last 16 values. + data = data + size - 16; + for (int64_t i = 0; i < 16; i++) { + data[i] = uniform_real(rand_uint32(), 0.f, 1.f); + } + normal_fill_16(data, mean, std); + } + } else { + // Strange handling, hard to understand, but keeping it consistent with PyTorch. + for (int64_t i = 0; i < size; i++) { + data[i] = (float)normal_double_value(mean, std); + } + } + } + +public: + MT19937RNG(uint64_t seed = 0) { manual_seed(seed); } + + void manual_seed(uint64_t seed) override { + s.seed_ = seed; + s.seeded_ = true; + s.state_[0] = (uint32_t)(seed & 0xffffffffU); + for (int j = 1; j < N; j++) { + uint32_t prev = s.state_[j - 1]; + s.state_[j] = 1812433253U * (prev ^ (prev >> 30)) + j; + } + s.left_ = 1; + s.next_ = 0; + s.has_next_gauss = false; + } + + std::vector randn(uint32_t n) override { + std::vector out; + out.resize(n); + randn((float*)out.data(), out.size()); + return out; + } +}; + +#endif // __RNG_MT19937_HPP__ \ No newline at end of file diff --git a/otherarch/sdcpp/sdtype_adapter.cpp b/otherarch/sdcpp/sdtype_adapter.cpp index cf0328fe1..003c77773 100644 --- a/otherarch/sdcpp/sdtype_adapter.cpp +++ b/otherarch/sdcpp/sdtype_adapter.cpp @@ -200,11 +200,6 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) { int lora_apply_mode = std::max(0, std::min(2, inputs.lora_apply_mode)); - if (inputs.quant > 0) - { - lora_apply_mode = LORA_APPLY_AT_RUNTIME; - } - if(lorafilename!="") { const char* lora_apply_mode_name = lora_apply_mode == 1 ? "immediately" diff --git a/otherarch/sdcpp/stable-diffusion.cpp b/otherarch/sdcpp/stable-diffusion.cpp index 18814fa1a..2dcb829a3 100644 --- a/otherarch/sdcpp/stable-diffusion.cpp +++ b/otherarch/sdcpp/stable-diffusion.cpp @@ -2,6 +2,7 @@ #include "model.h" #include "rng.hpp" +#include "rng_mt19937.hpp" #include "rng_philox.hpp" #include "stable-diffusion.h" #include "util.h" @@ -100,10 +101,11 @@ public: bool vae_decode_only = false; bool free_params_immediately = false; - std::shared_ptr rng = std::make_shared(); - int n_threads = -1; - float scale_factor = 0.18215f; - float shift_factor = 0.f; + std::shared_ptr rng = std::make_shared(); + std::shared_ptr sampler_rng = nullptr; + int n_threads = -1; + float scale_factor = 0.18215f; + float shift_factor = 0.f; std::shared_ptr cond_stage_model; std::shared_ptr clip_vision; // for svd or wan2.1 i2v @@ -200,6 +202,16 @@ public: } } + std::shared_ptr get_rng(rng_type_t rng_type) { + if (rng_type == STD_DEFAULT_RNG) { + return std::make_shared(); + } else if (rng_type == CPU_RNG) { + return std::make_shared(); + } else { // default: CUDA_RNG + return std::make_shared(); + } + } + bool init(const sd_ctx_params_t* sd_ctx_params) { n_threads = sd_ctx_params->n_threads; vae_decode_only = sd_ctx_params->vae_decode_only; @@ -209,10 +221,11 @@ public: use_tiny_autoencoder = taesd_path.size() > 0; offload_params_to_cpu = sd_ctx_params->offload_params_to_cpu; - if (sd_ctx_params->rng_type == STD_DEFAULT_RNG) { - rng = std::make_shared(); - } else if (sd_ctx_params->rng_type == CUDA_RNG) { - rng = std::make_shared(); + rng = get_rng(sd_ctx_params->rng_type); + if (sd_ctx_params->sampler_rng_type != RNG_TYPE_COUNT) { + sampler_rng = get_rng(sd_ctx_params->sampler_rng_type); + } else { + sampler_rng = rng; } ggml_log_set(ggml_log_callback_default, nullptr); @@ -422,11 +435,12 @@ public: } } - ggml_type wtype = (int)sd_ctx_params->wtype < std::min(SD_TYPE_COUNT, GGML_TYPE_COUNT) - ? (ggml_type)sd_ctx_params->wtype - : GGML_TYPE_COUNT; - if (wtype != GGML_TYPE_COUNT) { - model_loader.set_wtype_override(wtype); + ggml_type wtype = (int)sd_ctx_params->wtype < std::min(SD_TYPE_COUNT, GGML_TYPE_COUNT) + ? (ggml_type)sd_ctx_params->wtype + : GGML_TYPE_COUNT; + std::string tensor_type_rules = SAFE_STR(sd_ctx_params->tensor_type_rules); + if (wtype != GGML_TYPE_COUNT || tensor_type_rules.size() > 0) { + model_loader.set_wtype_override(wtype, tensor_type_rules); } std::map wtype_stat = model_loader.get_wtype_stat(); @@ -457,10 +471,14 @@ public: if (sd_ctx_params->lora_apply_mode == LORA_APPLY_AUTO) { bool have_quantized_weight = false; - for (const auto& [type, _] : wtype_stat) { - if (ggml_is_quantized(type)) { - have_quantized_weight = true; - break; + if (wtype != GGML_TYPE_COUNT && ggml_is_quantized(wtype)) { + have_quantized_weight = true; + } else { + for (const auto& [type, _] : wtype_stat) { + if (ggml_is_quantized(type)) { + have_quantized_weight = true; + break; + } } } if (have_quantized_weight) { @@ -1901,7 +1919,7 @@ public: return denoised; }; - sample_k_diffusion(method, denoise, work_ctx, x, sigmas, rng, eta); + sample_k_diffusion(method, denoise, work_ctx, x, sigmas, sampler_rng, eta); if (inverse_noise_scaling) { x = denoiser->inverse_noise_scaling(sigmas[sigmas.size() - 1], x); @@ -2300,6 +2318,7 @@ enum sd_type_t str_to_sd_type(const char* str) { const char* rng_type_to_str[] = { "std_default", "cuda", + "cpu", }; const char* sd_rng_type_name(enum rng_type_t rng_type) { @@ -2455,6 +2474,7 @@ void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params) { sd_ctx_params->n_threads = sd_get_num_physical_cores(); sd_ctx_params->wtype = SD_TYPE_COUNT; sd_ctx_params->rng_type = CUDA_RNG; + sd_ctx_params->sampler_rng_type = RNG_TYPE_COUNT; sd_ctx_params->prediction = DEFAULT_PRED; sd_ctx_params->lora_apply_mode = LORA_APPLY_AUTO; sd_ctx_params->offload_params_to_cpu = false; @@ -2490,11 +2510,13 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) { "lora_model_dir: %s\n" "embedding_dir: %s\n" "photo_maker_path: %s\n" + "tensor_type_rules: %s\n" "vae_decode_only: %s\n" "free_params_immediately: %s\n" "n_threads: %d\n" "wtype: %s\n" "rng_type: %s\n" + "sampler_rng_type: %s\n" "prediction: %s\n" "offload_params_to_cpu: %s\n" "keep_clip_on_cpu: %s\n" @@ -2519,11 +2541,13 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) { SAFE_STR(sd_ctx_params->lora_model_dir), SAFE_STR(sd_ctx_params->embedding_dir), SAFE_STR(sd_ctx_params->photo_maker_path), + SAFE_STR(sd_ctx_params->tensor_type_rules), BOOL_STR(sd_ctx_params->vae_decode_only), BOOL_STR(sd_ctx_params->free_params_immediately), sd_ctx_params->n_threads, sd_type_name(sd_ctx_params->wtype), sd_rng_type_name(sd_ctx_params->rng_type), + sd_rng_type_name(sd_ctx_params->sampler_rng_type), sd_prediction_name(sd_ctx_params->prediction), BOOL_STR(sd_ctx_params->offload_params_to_cpu), BOOL_STR(sd_ctx_params->keep_clip_on_cpu), @@ -2822,18 +2846,24 @@ sd_image_t* generate_image_internal(sd_ctx_t* sd_ctx, LOG_WARN("Turn off PhotoMaker"); sd_ctx->sd->stacked_id = false; } else { - id_cond.c_crossattn = sd_ctx->sd->id_encoder(work_ctx, init_img, id_cond.c_crossattn, id_embeds, class_tokens_mask); - int64_t t1 = ggml_time_ms(); - LOG_INFO("Photomaker ID Stacking, taking %" PRId64 " ms", t1 - t0); - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->pmid_model->free_params_buffer(); - } - // Encode input prompt without the trigger word for delayed conditioning - prompt_text_only = sd_ctx->sd->cond_stage_model->remove_trigger_from_prompt(work_ctx, prompt); - // printf("%s || %s \n", prompt.c_str(), prompt_text_only.c_str()); - prompt = prompt_text_only; // - if (sample_steps < 50) { - LOG_WARN("It's recommended to use >= 50 steps for photo maker!"); + if (pm_params.id_images_count != id_embeds->ne[1]) { + LOG_WARN("PhotoMaker image count (%d) does NOT match ID embeds (%d). You should run face_detect.py again.", pm_params.id_images_count, id_embeds->ne[1]); + LOG_WARN("Turn off PhotoMaker"); + sd_ctx->sd->stacked_id = false; + } else { + id_cond.c_crossattn = sd_ctx->sd->id_encoder(work_ctx, init_img, id_cond.c_crossattn, id_embeds, class_tokens_mask); + int64_t t1 = ggml_time_ms(); + LOG_INFO("Photomaker ID Stacking, taking %" PRId64 " ms", t1 - t0); + if (sd_ctx->sd->free_params_immediately) { + sd_ctx->sd->pmid_model->free_params_buffer(); + } + // Encode input prompt without the trigger word for delayed conditioning + prompt_text_only = sd_ctx->sd->cond_stage_model->remove_trigger_from_prompt(work_ctx, prompt); + // printf("%s || %s \n", prompt.c_str(), prompt_text_only.c_str()); + prompt = prompt_text_only; // + if (sample_steps < 50) { + LOG_WARN("It's recommended to use >= 50 steps for photo maker!"); + } } } } else { @@ -2979,6 +3009,7 @@ sd_image_t* generate_image_internal(sd_ctx_t* sd_ctx, LOG_INFO("generating image: %i/%i - seed %" PRId64, b + 1, batch_count, cur_seed); sd_ctx->sd->rng->manual_seed(cur_seed); + sd_ctx->sd->sampler_rng->manual_seed(cur_seed); struct ggml_tensor* x_t = init_latent; struct ggml_tensor* noise = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, W, H, C, 1); ggml_ext_im_set_randn_f32(noise, sd_ctx->sd->rng); @@ -3105,6 +3136,7 @@ sd_image_t* generate_image(sd_ctx_t* sd_ctx, const sd_img_gen_params_t* sd_img_g seed = rand(); } sd_ctx->sd->rng->manual_seed(seed); + sd_ctx->sd->sampler_rng->manual_seed(seed); int sample_steps = sd_img_gen_params->sample_params.sample_steps; @@ -3396,6 +3428,7 @@ SD_API sd_image_t* generate_video(sd_ctx_t* sd_ctx, const sd_vid_gen_params_t* s } sd_ctx->sd->rng->manual_seed(seed); + sd_ctx->sd->sampler_rng->manual_seed(seed); int64_t t0 = ggml_time_ms(); diff --git a/otherarch/sdcpp/stable-diffusion.h b/otherarch/sdcpp/stable-diffusion.h index f32579f64..03c701679 100644 --- a/otherarch/sdcpp/stable-diffusion.h +++ b/otherarch/sdcpp/stable-diffusion.h @@ -31,6 +31,7 @@ extern "C" { enum rng_type_t { STD_DEFAULT_RNG, CUDA_RNG, + CPU_RNG, RNG_TYPE_COUNT }; @@ -166,11 +167,13 @@ typedef struct { const char* lora_model_dir; const char* embedding_dir; const char* photo_maker_path; + const char* tensor_type_rules; bool vae_decode_only; bool free_params_immediately; int n_threads; enum sd_type_t wtype; enum rng_type_t rng_type; + enum rng_type_t sampler_rng_type; enum prediction_t prediction; enum lora_apply_mode_t lora_apply_mode; bool offload_params_to_cpu; diff --git a/otherarch/sdcpp/util.cpp b/otherarch/sdcpp/util.cpp index 658c4843a..004ad01ce 100644 --- a/otherarch/sdcpp/util.cpp +++ b/otherarch/sdcpp/util.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -567,6 +568,8 @@ sd_image_f32_t clip_preprocess(sd_image_f32_t image, int target_width, int targe // (abc) - increases attention to abc by a multiplier of 1.1 // (abc:3.12) - increases attention to abc by a multiplier of 3.12 // [abc] - decreases attention to abc by a multiplier of 1.1 +// BREAK - separates the prompt into conceptually distinct parts for sequential processing +// B - internal helper pattern; prevents 'B' in 'BREAK' from being consumed as normal text // \( - literal character '(' // \[ - literal character '[' // \) - literal character ')' @@ -602,7 +605,7 @@ std::vector> parse_prompt_attention(const std::str float round_bracket_multiplier = 1.1f; float square_bracket_multiplier = 1 / 1.1f; - std::regex re_attention(R"(\\\(|\\\)|\\\[|\\\]|\\\\|\\|\(|\[|:([+-]?[.\d]+)\)|\)|\]|[^\\()\[\]:]+|:)"); + std::regex re_attention(R"(\\\(|\\\)|\\\[|\\\]|\\\\|\\|\(|\[|:([+-]?[.\d]+)\)|\)|\]|\bBREAK\b|[^\\()\[\]:B]+|:|\bB)"); std::regex re_break(R"(\s*\bBREAK\b\s*)"); auto multiply_range = [&](int start_position, float multiplier) { @@ -611,7 +614,7 @@ std::vector> parse_prompt_attention(const std::str } }; - std::smatch m; + std::smatch m, m2; std::string remaining_text = text; while (std::regex_search(remaining_text, m, re_attention)) { @@ -635,6 +638,8 @@ std::vector> parse_prompt_attention(const std::str square_brackets.pop_back(); } else if (text == "\\(") { res.push_back({text.substr(1), 1.0f}); + } else if (std::regex_search(text, m2, re_break)) { + res.push_back({"BREAK", -1.0f}); } else { res.push_back({text, 1.0f}); }