diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f1067662..a85a6af13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## Update for 2023-11-24 +Note: Release pending `diffusers==0.24` + +- **Diffusers** + - **IP adapter** + - Lightweight implementation of T2I adapters which can guide generation towards specific image style + - Supports most T2I models, not limited to SD + - **HDR latent control**, based on [article](https://huggingface.co/blog/TimothyAlexisVass/explaining-the-sdxl-latent-space#long-prompts-at-high-guidance-scales-becoming-possible) + - In *Advanced* params + - Allows control of *latent clamping*, *color centering* and *range maximimization* + - Supported by *XYZ grid* - **General** - log level defaults to info for console and debug for log file - better prompt display in process tab diff --git a/modules/processing.py b/modules/processing.py index 3c3faada5..0710264d1 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -121,7 +121,7 @@ class StableDiffusionProcessing: """ The first set of paramaters: sd_models -> do_not_reload_embeddings represent the minimum required to create a StableDiffusionProcessing """ - def __init__(self, sd_model=None, outpath_samples=None, outpath_grids=None, prompt: str = "", styles: List[str] = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, seed_enable_extras: bool = True, sampler_name: str = None, latent_sampler: str = None, batch_size: int = 1, n_iter: int = 1, steps: int = 50, cfg_scale: float = 7.0, image_cfg_scale: float = None, clip_skip: int = 1, width: int = 512, height: int = 512, full_quality: bool = True, restore_faces: bool = False, tiling: bool = False, do_not_save_samples: bool = False, do_not_save_grid: bool = False, extra_generation_params: Dict[Any, Any] = None, overlay_images: Any = None, negative_prompt: str = None, eta: float = None, do_not_reload_embeddings: bool = False, denoising_strength: float = 0, diffusers_guidance_rescale: float = 0.7, override_settings: Dict[str, Any] = None, override_settings_restore_afterwards: bool = True, sampler_index: int = None, script_args: list = None): # pylint: disable=unused-argument + def __init__(self, sd_model=None, outpath_samples=None, outpath_grids=None, prompt: str = "", styles: List[str] = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, seed_enable_extras: bool = True, sampler_name: str = None, latent_sampler: str = None, batch_size: int = 1, n_iter: int = 1, steps: int = 50, cfg_scale: float = 7.0, image_cfg_scale: float = None, clip_skip: int = 1, width: int = 512, height: int = 512, full_quality: bool = True, restore_faces: bool = False, tiling: bool = False, do_not_save_samples: bool = False, do_not_save_grid: bool = False, extra_generation_params: Dict[Any, Any] = None, overlay_images: Any = None, negative_prompt: str = None, eta: float = None, do_not_reload_embeddings: bool = False, denoising_strength: float = 0, diffusers_guidance_rescale: float = 0.7, hdr_clamp: bool = False, hdr_boundary: float = 4.0, hdr_threshold: float = 3.5, hdr_center: bool = False, hdr_channel_shift: float = 0.8, hdr_full_shift: float = 0.8, hdr_maximize: bool = False, hdr_max_boundry: float = 4.0, override_settings: Dict[str, Any] = None, override_settings_restore_afterwards: bool = True, sampler_index: int = None, script_args: list = None): # pylint: disable=unused-argument self.outpath_samples: str = outpath_samples self.outpath_grids: str = outpath_grids @@ -203,6 +203,15 @@ class StableDiffusionProcessing: self.scripts_value: modules.scripts.ScriptRunner = field(default=None, init=False) self.script_args_value: list = field(default=None, init=False) self.scripts_setup_complete: bool = field(default=False, init=False) + # hdr + self.hdr_clamp = hdr_clamp + self.hdr_boundary = hdr_boundary + self.hdr_threshold = hdr_threshold + self.hdr_center = hdr_center + self.hdr_channel_shift = hdr_channel_shift + self.hdr_full_shift = hdr_full_shift + self.hdr_maximize = hdr_maximize + self.hdr_max_boundry = hdr_max_boundry @property diff --git a/modules/processing_correction.py b/modules/processing_correction.py new file mode 100644 index 000000000..5ddecc885 --- /dev/null +++ b/modules/processing_correction.py @@ -0,0 +1,57 @@ +""" +based on article by TimothyAlexisVass +https://huggingface.co/blog/TimothyAlexisVass/explaining-the-sdxl-latent-space +""" + +import os +import torch +from modules import shared + + +debug = shared.log.info if os.environ.get('SD_HDR_DEBUG', None) is not None else lambda *args, **kwargs: None + + +def soft_clamp_tensor(input_tensor, threshold=3.5, boundary=4): + # shrinking towards the mean; will also remove outliers + if max(abs(input_tensor.max()), abs(input_tensor.min())) < 4: + return input_tensor + channel_dim = 1 + max_vals = input_tensor.max(channel_dim, keepdim=True)[0] + max_replace = ((input_tensor - threshold) / (max_vals - threshold)) * (boundary - threshold) + threshold + over_mask = input_tensor > threshold + min_vals = input_tensor.min(channel_dim, keepdim=True)[0] + min_replace = ((input_tensor + threshold) / (min_vals + threshold)) * (-boundary + threshold) - threshold + under_mask = input_tensor < -threshold + debug(f'HDE soft clamp: threshold={threshold} boundary={boundary}') + res = torch.where(over_mask, max_replace, torch.where(under_mask, min_replace, input_tensor)) + return res + + +def center_tensor(input_tensor, channel_shift=1.0, full_shift=1.0, channels=[0, 1, 2, 3]): + means = [] + for channel in channels: + means.append(input_tensor[0, channel].mean()) + input_tensor[0, channel] -= means[-1] * channel_shift + debug(f'HDR center: channel-shift{channel_shift} full-shift={full_shift} means={torch.stack(means)}') + res = input_tensor - input_tensor.mean() * full_shift + return res + + +def maximize_tensor(input_tensor, boundary=4.0, channels=[0, 1, 2]): + min_val = input_tensor.min() + max_val = input_tensor.max() + normalization_factor = boundary / max(abs(min_val), abs(max_val)) + input_tensor[0, channels] *= normalization_factor + debug(f'HDR maximize: boundary={boundary} min={min_val} max={max_val} factor={normalization_factor}') + return input_tensor + + +def correction_callback(p, timestep, kwags): + if timestep > 950 and p.hdr_clamp: + kwags["latents"] = soft_clamp_tensor(kwags["latents"], threshold=p.hdr_threshold, boundary=p.hdr_boundary) + if timestep > 700 and p.hdr_center: + kwags["latents"] = center_tensor(kwags["latents"], channel_shift=p.hdr_channel_shift, full_shift=p.hdr_full_shift) + if timestep > 1 and timestep < 100 and p.hdr_maximize: + kwags["latents"] = center_tensor(kwags["latents"], channel_shift=0.6, full_shift=1.0) + kwags["latents"] = maximize_tensor(kwags["latents"], boundary=p.hdr_max_boundry) + return kwags diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 1e984134b..55cbdb690 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -17,6 +17,7 @@ import modules.errors as errors from modules.processing import StableDiffusionProcessing, create_random_tensors import modules.prompt_parser_diffusers as prompt_parser_diffusers from modules.sd_hijack_hypertile import hypertile_set +from modules.processing_correction import correction_callback def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_prompts): @@ -71,7 +72,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro raise AssertionError('Interrupted...') time.sleep(0.1) - def diffusers_callback(_pipe, step: int, _timestep: int, kwargs: dict): + def diffusers_callback(_pipe, step: int, timestep: int, kwargs: dict): + latents = correction_callback(p, timestep, kwargs) latents = kwargs['latents'] shared.state.sampling_step = step shared.state.current_latent = latents diff --git a/modules/txt2img.py b/modules/txt2img.py index 899af07db..3e51190cc 100644 --- a/modules/txt2img.py +++ b/modules/txt2img.py @@ -4,7 +4,7 @@ from modules.generation_parameters_copypaste import create_override_settings_dic from modules.ui import plaintext_to_html -def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, latent_index: int, full_quality: bool, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, diffusers_guidance_rescale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_force: bool, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_steps: int, refiner_start: int, refiner_prompt: str, refiner_negative: str, override_settings_texts, *args): # pylint: disable=unused-argument +def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, latent_index: int, full_quality: bool, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, diffusers_guidance_rescale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_force: bool, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_steps: int, refiner_start: int, refiner_prompt: str, refiner_negative: str, hdr_clamp, hdr_boundary, hdr_threshold, hdr_center, hdr_channel_shift, hdr_full_shift, hdr_maximize, hdr_max_boundry, override_settings_texts, *args): # pylint: disable=unused-argument shared.log.debug(f'txt2img: id_task={id_task}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|steps={steps}|sampler_index={sampler_index}|latent_index={latent_index}|full_quality={full_quality}|restore_faces={restore_faces}|tiling={tiling}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|clip_skip={clip_skip}|seed={seed}|subseed={subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}||height={height}|width={width}|enable_hr={enable_hr}|denoising_strength={denoising_strength}|hr_scale={hr_scale}|hr_upscaler={hr_upscaler}|hr_force={hr_force}|hr_second_pass_steps={hr_second_pass_steps}|hr_resize_x={hr_resize_x}|hr_resize_y={hr_resize_y}|image_cfg_scale={image_cfg_scale}|diffusers_guidance_rescale={diffusers_guidance_rescale}|refiner_steps={refiner_steps}|refiner_start={refiner_start}|refiner_prompt={refiner_prompt}|refiner_negative={refiner_negative}|override_settings_texts={override_settings_texts}') @@ -57,6 +57,9 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step refiner_start=refiner_start, refiner_prompt=refiner_prompt, refiner_negative=refiner_negative, + hdr_clamp=hdr_clamp, hdr_boundary=hdr_boundary, hdr_threshold=hdr_threshold, + hdr_center=hdr_center, hdr_channel_shift=hdr_channel_shift, hdr_full_shift=hdr_full_shift, + hdr_maximize=hdr_maximize, hdr_max_boundry=hdr_max_boundry, override_settings=override_settings, ) p.scripts = modules.scripts.scripts_txt2img diff --git a/modules/ui.py b/modules/ui.py index 0dcf9c25f..611d1c0ae 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -427,6 +427,17 @@ def create_ui(startup_timer = None): full_quality = gr.Checkbox(label='Full quality', value=True, elem_id="txt2img_full_quality") restore_faces = gr.Checkbox(label='Face restore', value=False, visible=len(modules.shared.face_restorers) > 1, elem_id="txt2img_restore_faces") tiling = gr.Checkbox(label='Tiling', value=False, elem_id="txt2img_tiling") + with FormRow(): + hdr_clamp = gr.Checkbox(label='HDR clamp', value=False, elem_id="txt2img_hdr_clamp") + hdr_boundary = gr.Slider(minimum=0.0, maximum=10.0, step=0.1, value=4.0, label='Range', elem_id="txt2img_hdr_boundary") + hdr_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, value=0.95, label='Threshold', elem_id="txt2img_hdr_threshold") + with FormRow(): + hdr_center = gr.Checkbox(label='HDR center', value=False, elem_id="txt2img_hdr_center") + hdr_channel_shift = gr.Slider(minimum=0.0, maximum=2.0, step=0.1, value=1.0, label='Channel shift', elem_id="txt2img_hdr_channel_shift") + hdr_full_shift = gr.Slider(minimum=0.0, maximum=2.0, step=0.1, value=1, label='Full shift', elem_id="txt2img_hdr_full_shift") + with FormRow(): + hdr_maximize = gr.Checkbox(label='HDR maximize', value=False, elem_id="txt2img_hdr_maximize") + hdr_max_boundry = gr.Slider(minimum=0.0, maximum=10.0, step=0.1, value=4.0, label='range', elem_id="txt2img_hdr_max_boundry") with gr.Accordion(open=False, label="Second pass", elem_id="txt2img_second_pass", elem_classes=["small-accordion"]): with FormGroup(): @@ -493,6 +504,7 @@ def create_ui(startup_timer = None): enable_hr, denoising_strength, hr_scale, hr_upscaler, hr_force, hr_second_pass_steps, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative, + hdr_clamp, hdr_boundary, hdr_threshold, hdr_center, hdr_channel_shift, hdr_full_shift, hdr_maximize, hdr_max_boundry, override_settings, ] + custom_inputs, outputs=[ diff --git a/scripts/ipadapter.py b/scripts/ipadapter.py new file mode 100644 index 000000000..b9612e44c --- /dev/null +++ b/scripts/ipadapter.py @@ -0,0 +1,96 @@ +""" +lightweight ip-adapter applied to existing pipeline +- downloads image_encoder or first usage (2.5GB) +- introduced via: https://github.com/huggingface/diffusers/pull/5713 +- ip adapters: https://huggingface.co/h94/IP-Adapter +""" + +import gradio as gr +from modules import scripts, processing + + +image_encoder = None +ADAPTERS = [ + 'none', + 'models/ip-adapter_sd15', + 'models/ip-adapter_sd15_light', + # 'models/ip-adapter_sd15_vit-G', # RuntimeError: mat1 and mat2 shapes cannot be multiplied (2x1024 and 1280x3072) + # 'models/ip-adapter-plus_sd15', # KeyError: 'proj.weight' + # 'models/ip-adapter-plus-face_sd15', # KeyError: 'proj.weight' + # 'models/ip-adapter-full-face_sd15', # KeyError: 'proj.weight' + 'sdxl_models/ip-adapter_sdxl', + # 'sdxl_models/ip-adapter_sdxl_vit-h', + # 'sdxl_models/ip-adapter-plus_sdxl_vit-h', + # 'sdxl_models/ip-adapter-plus-face_sdxl_vit-h', +] + + +# main processing used in both modes +def before_process(p: processing.StableDiffusionProcessing, adapter, scale, image): + import torch + import transformers + from modules import shared, devices + + # init code + if shared.sd_model is None: + return + if adapter == 'none' or image is None: + if hasattr(shared.sd_model, 'set_ip_adapter_scale'): + shared.sd_model.set_ip_adapter_scale(0) + return + if shared.backend != shared.Backend.DIFFUSERS: + shared.log.warning('IP adapter: not in diffusers mode') + return + if not hasattr(shared.sd_model, 'load_ip_adapter'): + shared.log.error(f'IP adapter: pipeline not supported: {shared.sd_model.__class__.__name__}') + return + if getattr(shared.sd_model, 'image_encoder', None) is None: + if shared.sd_model_type == 'sd': + subfolder = 'models/image_encoder' + elif shared.sd_model_type == 'sdxl': + subfolder = 'sdxl_models/image_encoder' + else: + shared.log.error(f'IP adapter: unsupported model type: {shared.sd_model_type}') + return + global image_encoder # pylint: disable=global-statement + if image_encoder is None: + try: + image_encoder = transformers.CLIPVisionModelWithProjection.from_pretrained("h94/IP-Adapter", subfolder=subfolder, torch_dtype=torch.float16, cache_dir=shared.opts.diffusers_dir, use_safetensors=True).to(devices.device) + except Exception as e: + shared.log.error(f'IP adapter: failed to load image encoder: {e}') + return + + # main code + subfolder, model = adapter.split('/') + shared.log.info(f'IP adapter: scale={scale} adapter="{model}" image={image}') + shared.sd_model.image_encoder = image_encoder + shared.sd_model.load_ip_adapter("h94/IP-Adapter", subfolder=subfolder, weight_name=f'{model}.safetensors') + shared.sd_model.set_ip_adapter_scale(scale) + p.task_args = { 'ip_adapter_image': image } + p.extra_generation_params["IP Adapter"] = f'{adapter}:{scale}' + + +# defines script for dual-mode usage +class Script(scripts.Script): + # see below for all available options and callbacks + # + + def title(self): + return 'IP Adapter' + + def show(self, is_img2img): + return scripts.AlwaysVisible + + # return signature is array of gradio components + def ui(self, _is_img2img): + with gr.Accordion('IP Adapter', open=False, elem_id='ipadapter'): + with gr.Row(): + adapter = gr.Dropdown(label='Adapter', choices=ADAPTERS, value='none') + scale = gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=0.5) + with gr.Row(): + image = gr.Image(image_mode='RGB', label='Image', source='upload', type='pil', width=512) + return [adapter, scale, image] + + # triggered by callback + def before_process(self, p: processing.StableDiffusionProcessing, *args): # pylint: disable=arguments-differ + before_process(p, *args) diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index ad8cb65af..1ca2d838c 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -257,6 +257,11 @@ axis_options = [ AxisOption("[Refiner] Model", str, apply_refiner, fmt=format_value, cost=1.0, choices=lambda: ['None'] + sorted(sd_models.checkpoints_list)), AxisOption("[Refiner] Refiner start", float, apply_field("refiner_start")), AxisOption("[Refiner] Refiner steps", float, apply_field("refiner_steps")), + AxisOption("[HDR] Clamp boundary", float, apply_field("hdr_boundary")), + AxisOption("[HDR] Clamp threshold", float, apply_field("hdr_threshold")), + AxisOption("[HDR] Center channel shift", float, apply_field("hdr_channel_shift")), + AxisOption("[HDR] Center full shift", float, apply_field("hdr_full_shift")), + AxisOption("[HDR] Maximize boundary", float, apply_field("hdr_max_boundry")), AxisOption("[ToMe] Token merging ratio (txt2img)", float, apply_override('token_merging_ratio')), AxisOption("[ToMe] Token merging ratio (hires)", float, apply_override('token_merging_ratio_hr')), AxisOption("[FreeU] 1st stage backbone factor", float, apply_setting('freeu_b1')),