Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-20 01:31:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

added photomaker face cloning

Browse Source
This commit is contained in:
Concedo
2025-06-20 21:33:36 +08:00
parent 21881a861d
commit 4e40f2aaf4
6 changed files with 161 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
otherarch/sdcpp/conditioner.hpp
+1
View File
@@ -597,6 +597,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
GGML_ASSERT(it != tokens.end()); // prompt must have trigger word
tokens.erase(it);
return decode(tokens);
//return prompt; //kcpp we don't care about photomaker trigger words
}
SDCondition get_learned_condition(ggml_context* work_ctx,
otherarch/sdcpp/sdtype_adapter.cpp
+51 -11
View File
@@ -57,7 +57,7 @@ struct SDParams {
std::string controlnet_path;
std::string embeddings_path;
std::string stacked_id_embeddings_path;
std::string input_id_images_path;
std::string input_id_images_path = "";
sd_type_t wtype = SD_TYPE_COUNT;
std::string lora_model_dir;
std::string output_path = "output.png";
@@ -116,6 +116,7 @@ static int sddebugmode = 0;
static std::string recent_data = "";
static uint8_t * input_image_buffer = NULL;
static uint8_t * input_mask_buffer = NULL;
static uint8_t * input_photomaker_buffer = NULL;
static std::string sdplatformenv, sddeviceenv, sdvulkandeviceenv;
static bool notiling = false;
@@ -134,6 +135,7 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) {
std::string t5xxl_filename = inputs.t5xxl_filename;
std::string clipl_filename = inputs.clipl_filename;
std::string clipg_filename = inputs.clipg_filename;
std::string photomaker_filename = inputs.photomaker_filename;
notiling = inputs.notile;
cfg_side_limit = inputs.img_hard_limit;
cfg_square_limit = inputs.img_soft_limit;
@@ -164,6 +166,10 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) {
{
printf("With Custom Clip-G Model: %s\n",clipg_filename.c_str());
}
if(photomaker_filename!="")
{
printf("With PhotoMaker Model: %s\n",photomaker_filename.c_str());
}
if(inputs.quant)
{
printf("Note: Loading a pre-quantized model is always faster than using compress weights!\n");
@@ -205,6 +211,7 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) {
sd_params->t5xxl_path = t5xxl_filename;
sd_params->clip_l_path = clipl_filename;
sd_params->clip_g_path = clipg_filename;
sd_params->stacked_id_embeddings_path = photomaker_filename;
//if t5 is set, and model is a gguf, load it as a diffusion model path
bool endswithgguf = (sd_params->model_path.rfind(".gguf") == sd_params->model_path.size() - 5);
if(sd_params->t5xxl_path!="" && endswithgguf)
@@ -423,6 +430,7 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
std::string cleannegprompt = clean_input_prompt(inputs.negative_prompt);
std::string img2img_data = std::string(inputs.init_images);
std::string img2img_mask = std::string(inputs.mask);
std::string photomaker_image_data = std::string(inputs.photomaker_image);
std::string sampler = inputs.sample_method;
sd_params->prompt = cleanprompt;
@@ -490,15 +498,17 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
//for img2img
sd_image_t input_image = {0,0,0,nullptr};
sd_image_t photomaker_reference = {0,0,0,nullptr};
std::vector<uint8_t> image_buffer;
std::vector<uint8_t> image_mask_buffer;
std::vector<uint8_t> photomaker_buffer;
int nx, ny, nc;
int nx2, ny2, nc2;
int img2imgW = sd_params->width; //for img2img input
int img2imgH = sd_params->height;
int img2imgC = 3; // Assuming RGB image
std::vector<uint8_t> resized_image_buf(img2imgW * img2imgH * img2imgC);
std::vector<uint8_t> resized_mask_buf(img2imgW * img2imgH * img2imgC);
std::vector<uint8_t> resized_photomaker_buf(img2imgW * img2imgH * img2imgC);
std::string ts = get_timestamp_str();
if(!sd_is_quiet)
@@ -543,6 +553,35 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
sd_params->sample_method = sample_method_t::EULER_A;
}
if(photomaker_image_data!="")
{
if(input_photomaker_buffer!=nullptr) //just in time free old buffer
{
stbi_image_free(input_photomaker_buffer);
input_photomaker_buffer = nullptr;
}
int nx2, ny2, nc2;
photomaker_buffer = kcpp_base64_decode(photomaker_image_data);
input_photomaker_buffer = stbi_load_from_memory(photomaker_buffer.data(), photomaker_buffer.size(), &nx2, &ny2, &nc2, 1);
// Resize the image
int resok = stbir_resize_uint8(input_photomaker_buffer, nx2, ny2, 0, resized_photomaker_buf.data(), img2imgW, img2imgH, 0, 1);
if (!resok) {
printf("\nKCPP SD: resize photomaker image failed!\n");
output.data = "";
output.status = 0;
return output;
}
photomaker_reference.width = img2imgW;
photomaker_reference.height = img2imgH;
photomaker_reference.channel = img2imgC;
photomaker_reference.data = resized_photomaker_buf.data();
//ensure prompt has img keyword, otherwise append it
if (sd_params->prompt.find("img") == std::string::npos) {
sd_params->prompt += " img";
}
}
if (sd_params->mode == TXT2IMG) {
if(!sd_is_quiet && sddebugmode==1)
@@ -585,7 +624,8 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
sd_params->skip_layers.size(),
sd_params->slg_scale,
sd_params->skip_layer_start,
sd_params->skip_layer_end);
sd_params->skip_layer_end,
(photomaker_image_data!=""?&photomaker_reference:nullptr));
} else {
if (sd_params->width <= 0 || sd_params->width % 64 != 0 || sd_params->height <= 0 || sd_params->height % 64 != 0) {
@@ -596,18 +636,11 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
}
image_buffer = kcpp_base64_decode(img2img_data);
if(input_image_buffer!=nullptr) //just in time free old buffer
{
stbi_image_free(input_image_buffer);
input_image_buffer = nullptr;
}
if(input_mask_buffer!=nullptr) //just in time free old buffer
{
stbi_image_free(input_mask_buffer);
input_mask_buffer = nullptr;
}
input_image_buffer = stbi_load_from_memory(image_buffer.data(), image_buffer.size(), &nx, &ny, &nc, 3);
if (nx < 64 || ny < 64 || nx > 1024 || ny > 1024 || nc!= 3) {
@@ -634,6 +667,12 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
if(img2img_mask!="")
{
int nx2, ny2, nc2;
if(input_mask_buffer!=nullptr) //just in time free old buffer
{
stbi_image_free(input_mask_buffer);
input_mask_buffer = nullptr;
}
image_mask_buffer = kcpp_base64_decode(img2img_mask);
input_mask_buffer = stbi_load_from_memory(image_mask_buffer.data(), image_mask_buffer.size(), &nx2, &ny2, &nc2, 1);
// Resize the image
@@ -709,7 +748,8 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
sd_params->skip_layers.size(),
sd_params->slg_scale,
sd_params->skip_layer_start,
sd_params->skip_layer_end);
sd_params->skip_layer_end,
(photomaker_image_data!=""?&photomaker_reference:nullptr));
}
if (results == NULL) {
otherarch/sdcpp/stable-diffusion.cpp
+40 -6
View File
@@ -328,7 +328,7 @@ public:
LOG_WARN(
"!!!It looks like you are using SDXL model. "
"If you find that the generated images are completely black, "
"try specifying SDXL VAE FP16 Fix with the --vae parameter. "
"try specifying a different VAE. "
"You can find it here: https://huggingface.co/madebyollin/sdxl-vae-fp16-fix/blob/main/sdxl_vae.safetensors");
}
} else if (sd_version_is_sd3(version)) {
@@ -1408,7 +1408,8 @@ sd_image_t* generate_image(sd_ctx_t* sd_ctx,
float slg_scale = 0,
float skip_layer_start = 0.01,
float skip_layer_end = 0.2,
ggml_tensor* masked_image = NULL) {
ggml_tensor* masked_image = NULL,
const sd_image_t* photomaker_reference = nullptr) {
if (seed < 0) {
// Generally, when using the provided command line, the seed is always >0.
// However, to prevent potential issues if 'stable-diffusion.cpp' is invoked as a library
@@ -1451,6 +1452,10 @@ sd_image_t* generate_image(sd_ctx_t* sd_ctx,
ggml_tensor* init_img = NULL;
SDCondition id_cond;
std::vector<bool> class_tokens_mask;
if (sd_ctx->sd->pmid_model && photomaker_reference!=nullptr)
{
sd_ctx->sd->stacked_id = true; //turn on photomaker if needed
}
if (sd_ctx->sd->stacked_id) {
if (!sd_ctx->sd->pmid_lora->applied) {
t0 = ggml_time_ms();
@@ -1493,6 +1498,30 @@ sd_image_t* generate_image(sd_ctx_t* sd_ctx,
input_id_images.push_back(input_image);
}
}
// handle single photomaker image passed in by kcpp
if (sd_ctx->sd->pmid_model && photomaker_reference!=nullptr)
{
int c = 0;
int width, height;
width = photomaker_reference->width;
height = photomaker_reference->height;
c = photomaker_reference->channel;
uint8_t* input_image_buffer = photomaker_reference->data;
sd_image_t* input_image = NULL;
input_image = new sd_image_t{(uint32_t)width,
(uint32_t)height,
3,
input_image_buffer};
input_image = preprocess_id_image(input_image);
if (input_image == NULL) {
LOG_ERROR("\npreprocess input id image from kcpp photomaker failed\n");
} else {
LOG_INFO("\nPhotoMaker loaded image from kcpp\n");
input_id_images.push_back(input_image);
}
}
if (input_id_images.size() > 0) {
sd_ctx->sd->pmid_model->style_strength = style_ratio;
int32_t w = input_id_images[0]->width;
@@ -1744,7 +1773,8 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx,
size_t skip_layers_count = 0,
float slg_scale = 0,
float skip_layer_start = 0.01,
float skip_layer_end = 0.2) {
float skip_layer_end = 0.2,
const sd_image_t* photomaker_reference = nullptr) {
std::vector<int> skip_layers_vec(skip_layers, skip_layers + skip_layers_count);
LOG_DEBUG("txt2img %dx%d", width, height);
if (sd_ctx == NULL) {
@@ -1822,7 +1852,9 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx,
skip_layers_vec,
slg_scale,
skip_layer_start,
skip_layer_end);
skip_layer_end,
nullptr,
photomaker_reference);
size_t t1 = ggml_time_ms();
@@ -1856,7 +1888,8 @@ sd_image_t* img2img(sd_ctx_t* sd_ctx,
size_t skip_layers_count = 0,
float slg_scale = 0,
float skip_layer_start = 0.01,
float skip_layer_end = 0.2) {
float skip_layer_end = 0.2,
const sd_image_t* photomaker_reference = nullptr) {
std::vector<int> skip_layers_vec(skip_layers, skip_layers + skip_layers_count);
LOG_DEBUG("img2img %dx%d", width, height);
if (sd_ctx == NULL) {
@@ -2002,7 +2035,8 @@ sd_image_t* img2img(sd_ctx_t* sd_ctx,
slg_scale,
skip_layer_start,
skip_layer_end,
masked_image);
masked_image,
photomaker_reference);
size_t t2 = ggml_time_ms();