diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a4a0a416..ad77bbbe5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ ## Update for 2024-11-23 +- [Flux Tools](https://blackforestlabs.ai/flux-1-tools/): + **Redux** is actually a tool, **Fill** is inpaint/outpaint optimized version of *Flux-dev* + **Canny** & **Depth** are optimized versions of *Flux-dev* for their respective tasks: they are *not* ControlNets that work on top of a model + To use, go to image or control interface and select *Flux Tools* in scripts + All models are auto-downloaded on first use + *note*: All models are [gated](https://github.com/vladmandic/automatic/wiki/Gated) and require acceptance of terms and conditions via web page + *recommended*: Enable on-the-fly [quantization](https://github.com/vladmandic/automatic/wiki/Quantization) or [compression](https://github.com/vladmandic/automatic/wiki/NNCF-Compression) to reduce resource usage + *todo*: support for Canny/Depth LoRAs + - [Redux](https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev): ~0.1GB + works together with existing model and basically uses input image to analyze it and use that instead of prompt + *recommended*: low denoise strength levels result in more variety + - [Fill](https://huggingface.co/black-forest-labs/FLUX.1-Fill-dev): ~23.8GB, replaces currently loaded model + *note*: can be used in inpaint/outpaint mode only + - [Canny](https://huggingface.co/black-forest-labs/FLUX.1-Canny-dev): ~23.8GB, replaces currently loaded model + *recommended*: guidance scale 30 + - [Depth](https://huggingface.co/black-forest-labs/FLUX.1-Depth-dev): ~23.8GB, replaces currently loaded model + *recommended*: guidance scale 10 - Model loader improvements: - detect model components on model load fail - Flux, SD35: force unload model @@ -11,14 +28,15 @@ - Flux: do not recast quants - Sampler improvements - update DPM FlowMatch samplers -- UI: +- UI: - browser->server logging - Fixes: - update `diffusers` - fix README links - fix sdxl controlnet single-file loader - relax settings validator - - improve js progress calls resiliency + - improve js progress calls resiliency + - fix text-to-video pipeline ## Update for 2024-11-21 diff --git a/TODO.md b/TODO.md index 973e062dc..73008039d 100644 --- a/TODO.md +++ b/TODO.md @@ -7,9 +7,9 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - SD35 IPAdapter: - SD35 LoRA: - Flux IPAdapter: -- Flux Fill/ControlNet/Redux: - Flux NF4: - SANA: +- LTX-Video: ## Other diff --git a/installer.py b/installer.py index 8f552526a..396b53fab 100644 --- a/installer.py +++ b/installer.py @@ -212,7 +212,7 @@ def installed(package, friendly: str = None, reload = False, quiet = False): if friendly: pkgs = friendly.split() else: - pkgs = [p for p in package.split() if not p.startswith('-') and not p.startswith('=')] + pkgs = [p for p in package.split() if not p.startswith('-') and not p.startswith('=') and not p.startswith('git+')] pkgs = [p.split('/')[-1] for p in pkgs] # get only package name if installing from url for pkg in pkgs: if '!=' in pkg: @@ -295,7 +295,7 @@ def install(package, friendly: str = None, ignore: bool = False, reinstall: bool quick_allowed = False if args.reinstall or reinstall or not installed(package, friendly, quiet=quiet): deps = '' if not no_deps else '--no-deps ' - res = pip(f"install{' --upgrade' if not args.uv else ''} {deps}{package}", ignore=ignore, uv=package != "uv") + res = pip(f"install{' --upgrade' if not args.uv else ''} {deps}{package}", ignore=ignore, uv=package != "uv" and not package.startswith('git+')) try: import importlib # pylint: disable=deprecated-module importlib.reload(pkg_resources) diff --git a/javascript/logger.js b/javascript/logger.js index 4ff1fb822..5aa8face3 100644 --- a/javascript/logger.js +++ b/javascript/logger.js @@ -19,7 +19,8 @@ const error = async (...msg) => { const ts = `${dt.getHours().toString().padStart(2, '0')}:${dt.getMinutes().toString().padStart(2, '0')}:${dt.getSeconds().toString().padStart(2, '0')}.${dt.getMilliseconds().toString().padStart(3, '0')}`; if (window.logger) window.logger.innerHTML += window.logPrettyPrint(...msg); console.error(ts, ...msg); // eslint-disable-line no-console - xhrPost('/sdapi/v1/log', { error: msg.join(' ') }); // eslint-disable-line no-use-before-define + const txt = msg.join(' '); + if (!txt.includes('asctime') && !txt.includes('xhr.')) xhrPost('/sdapi/v1/log', { error: txt }); // eslint-disable-line no-use-before-define }; const xhrInternal = (xhrObj, data, handler = undefined, errorHandler = undefined, ignore = false) => { diff --git a/modules/model_flux.py b/modules/model_flux.py index 17234d9a4..324e50b36 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -306,9 +306,17 @@ def load_flux(checkpoint_info, diffusers_load_config): # triggered by opts.sd_ch model_te.loaded_te = shared.opts.sd_text_encoder if vae is not None: kwargs['vae'] = vae - shared.log.debug(f'Load model: type=FLUX preloaded={list(kwargs)}') if repo_id == 'sayakpaul/flux.1-dev-nf4': repo_id = 'black-forest-labs/FLUX.1-dev' # workaround since sayakpaul model is missing model_index.json + if 'Fill' in repo_id: + cls = diffusers.FluxFillPipeline + elif 'Canny' in repo_id: + cls = diffusers.FluxControlPipeline + elif 'Depth' in repo_id: + cls = diffusers.FluxControlPipeline + else: + cls = diffusers.FluxPipeline + shared.log.debug(f'Load model: type=FLUX cls={cls.__name__} preloaded={list(kwargs)} revision={diffusers_load_config.get("revision", None)}') for c in kwargs: if kwargs[c].dtype == torch.float32 and devices.dtype != torch.float32: shared.log.warning(f'Load model: type=FLUX component={c} dtype={kwargs[c].dtype} cast dtype={devices.dtype} recast') @@ -319,7 +327,7 @@ def load_flux(checkpoint_info, diffusers_load_config): # triggered by opts.sd_ch if checkpoint_info.path.endswith('.safetensors') and os.path.isfile(checkpoint_info.path): pipe = diffusers.FluxPipeline.from_single_file(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **kwargs, **diffusers_load_config) else: - pipe = diffusers.FluxPipeline.from_pretrained(repo_id, cache_dir=shared.opts.diffusers_dir, **kwargs, **diffusers_load_config) + pipe = cls.from_pretrained(repo_id, cache_dir=shared.opts.diffusers_dir, **kwargs, **diffusers_load_config) # release memory transformer = None diff --git a/modules/modelloader.py b/modules/modelloader.py index ce36a739b..b1b3930d6 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -326,6 +326,9 @@ def find_diffuser(name: str, full=False): return [repo[0]['name']] hf_api = hf.HfApi() models = list(hf_api.list_models(model_name=name, library=['diffusers'], full=True, limit=20, sort="downloads", direction=-1)) + if len(models) == 0: + models = list(hf_api.list_models(model_name=name, full=True, limit=20, sort="downloads", direction=-1)) # widen search + models = [m for m in models if m.id.startswith(name)] # filter exact shared.log.debug(f'Searching diffusers models: {name} {len(models) > 0}') if len(models) > 0: if not full: diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 2164134b1..44dff811b 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -231,7 +231,8 @@ def process_hires(p: processing.StableDiffusionProcessing, output): output = shared.sd_model(**hires_args) # pylint: disable=not-callable if isinstance(output, dict): output = SimpleNamespace(**output) - shared.history.add(output.images, info=processing.create_infotext(p), ops=p.ops) + if hasattr(output, 'images'): + shared.history.add(output.images, info=processing.create_infotext(p), ops=p.ops) sd_models_compile.check_deepcache(enable=False) sd_models_compile.openvino_post_compile(op="base") except AssertionError as e: @@ -313,7 +314,8 @@ def process_refine(p: processing.StableDiffusionProcessing, output): output = shared.sd_refiner(**refiner_args) # pylint: disable=not-callable if isinstance(output, dict): output = SimpleNamespace(**output) - shared.history.add(output.images, info=processing.create_infotext(p), ops=p.ops) + if hasattr(output, 'images'): + shared.history.add(output.images, info=processing.create_infotext(p), ops=p.ops) sd_models_compile.openvino_post_compile(op="refiner") except AssertionError as e: shared.log.info(e) diff --git a/modules/sd_checkpoint.py b/modules/sd_checkpoint.py index afc5842e4..a4d84192f 100644 --- a/modules/sd_checkpoint.py +++ b/modules/sd_checkpoint.py @@ -198,8 +198,9 @@ def get_closet_checkpoint_match(s: str): if shared.opts.sd_checkpoint_autodownload and s.count('/') == 1: modelloader.hf_login() found = modelloader.find_diffuser(s, full=True) + found = [f for f in found if f == s] shared.log.info(f'HF search: model="{s}" results={found}') - if found is not None and len(found) == 1 and found[0] == s: + if found is not None and len(found) == 1: checkpoint_info = CheckpointInfo(s) checkpoint_info.type = 'huggingface' return checkpoint_info diff --git a/modules/sd_models.py b/modules/sd_models.py index a6ff19b6f..2ad204b46 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -771,7 +771,7 @@ def load_diffuser_file(model_type, pipeline, checkpoint_info, diffusers_load_con return sd_model -def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): # pylint: disable=unused-argument +def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model', revision=None): # pylint: disable=unused-argument if timer is None: timer = Timer() logging.getLogger("diffusers").setLevel(logging.ERROR) @@ -784,6 +784,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No "requires_safety_checker": False, # sd15 specific but we cant know ahead of time # "use_safetensors": True, } + if revision is not None: + diffusers_load_config['revision'] = revision if shared.opts.diffusers_model_load_variant != 'default': diffusers_load_config['variant'] = shared.opts.diffusers_model_load_variant if shared.opts.diffusers_pipeline == 'Custom Diffusers Pipeline' and len(shared.opts.custom_diffusers_pipeline) > 0: @@ -1077,6 +1079,8 @@ def set_diffuser_pipe(pipe, new_pipe_type): 'OmniGenPipeline', 'StableDiffusion3ControlNetPipeline', 'InstantIRPipeline', + 'FluxFillPipeline', + 'FluxControlPipeline', ] n = getattr(pipe.__class__, '__name__', '') @@ -1345,7 +1349,7 @@ def reload_text_encoder(initial=False): set_t5(pipe=shared.sd_model, module='text_encoder_3', t5=shared.opts.sd_text_encoder, cache_dir=shared.opts.diffusers_dir) -def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model', force=False): +def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model', force=False, revision=None): load_dict = shared.opts.sd_model_dict != model_data.sd_dict from modules import lowvram, sd_hijack checkpoint_info = info or select_checkpoint(op=op) # are we selecting model or dictionary @@ -1390,7 +1394,7 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model', load_model(checkpoint_info, already_loaded_state_dict=state_dict, timer=timer, op=op) model_data.sd_dict = shared.opts.sd_model_dict else: - load_diffuser(checkpoint_info, already_loaded_state_dict=state_dict, timer=timer, op=op) + load_diffuser(checkpoint_info, already_loaded_state_dict=state_dict, timer=timer, op=op, revision=revision) if load_dict and next_checkpoint_info is not None: model_data.sd_dict = shared.opts.sd_model_dict shared.opts.data["sd_model_checkpoint"] = next_checkpoint_info.title diff --git a/scripts/animatediff.py b/scripts/animatediff.py index 4c50f9cf6..91db60915 100644 --- a/scripts/animatediff.py +++ b/scripts/animatediff.py @@ -189,7 +189,7 @@ def set_free_noise(frames): class Script(scripts.Script): def title(self): - return 'Video AnimateDiff' + return 'Video: AnimateDiff' def show(self, is_img2img): # return scripts.AlwaysVisible if shared.native else False diff --git a/scripts/cogvideo.py b/scripts/cogvideo.py index 7f2c7225e..c988c05c4 100644 --- a/scripts/cogvideo.py +++ b/scripts/cogvideo.py @@ -22,7 +22,7 @@ debug = (os.environ.get('SD_LOAD_DEBUG', None) is not None) or (os.environ.get(' class Script(scripts.Script): def title(self): - return 'Video CogVideoX' + return 'Video: CogVideoX' def show(self, is_img2img): return shared.native diff --git a/scripts/flux_tools.py b/scripts/flux_tools.py new file mode 100644 index 000000000..9a2fdbd63 --- /dev/null +++ b/scripts/flux_tools.py @@ -0,0 +1,115 @@ +# https://github.com/huggingface/diffusers/pull/9985 + +import time +import gradio as gr +import diffusers +from modules import scripts, processing, shared, devices, sd_models +from installer import install + + +redux_pipe: diffusers.FluxPriorReduxPipeline = None +processor_canny = None +processor_depth = None +title = 'Flux Tools' + + +class Script(scripts.Script): + def title(self): + return f'{title}' + + def show(self, is_img2img): + return is_img2img if shared.native else False + + def ui(self, _is_img2img): # ui elements + with gr.Row(): + gr.HTML('  Flux.1 Redux
') + with gr.Row(): + tool = gr.Dropdown(label='Tool', choices=['None', 'Redux', 'Fill', 'Canny', 'Depth'], value='None') + strength = gr.Checkbox(label='Override denoise strength', value=True) + return [tool, strength] + + def run(self, p: processing.StableDiffusionProcessing, tool: str = 'None', strength: bool = True): # pylint: disable=arguments-differ + global redux_pipe, processor_canny, processor_depth # pylint: disable=global-statement + if tool is None or tool == 'None': + return + supported_model_list = ['f1'] + if shared.sd_model_type not in supported_model_list: + shared.log.warning(f'{title}: class={shared.sd_model.__class__.__name__} model={shared.sd_model_type} required={supported_model_list}') + return None + image = getattr(p, 'init_images', None) + if image is None or len(image) == 0: + shared.log.error(f'{title}: tool={tool} no init_images') + return None + else: + image = image[0] if isinstance(image, list) else image + + shared.log.info(f'{title}: tool={tool} init') + + t0 = time.time() + if tool == 'Redux': + # pipe_prior_redux = FluxPriorReduxPipeline.from_pretrained("black-forest-labs/FLUX.1-Redux-dev", revision="refs/pr/8", torch_dtype=torch.bfloat16).to("cuda") + if redux_pipe is None: + redux_pipe = diffusers.FluxPriorReduxPipeline.from_pretrained( + "black-forest-labs/FLUX.1-Redux-dev", + revision="refs/pr/8", + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir + ).to(devices.device) + redux_output = redux_pipe(image) + for k, v in redux_output.items(): + p.task_args[k] = v + else: + if redux_pipe is not None: + shared.log.debug(f'{title}: tool=Redux unload') + redux_pipe = None + + if tool == 'Fill': + # pipe = FluxFillPipeline.from_pretrained("black-forest-labs/FLUX.1-Fill-dev", torch_dtype=torch.bfloat16, revision="refs/pr/4").to("cuda") + if p.image_mask is None: + shared.log.error(f'{title}: tool={tool} no image_mask') + return None + if shared.sd_model.__class__.__name__ != 'FluxFillPipeline': + shared.opts.data["sd_model_checkpoint"] = "black-forest-labs/FLUX.1-Fill-dev" + sd_models.reload_model_weights(op='model', revision="refs/pr/4") + p.task_args['image'] = image + p.task_args['mask_image'] = p.image_mask + + if tool == 'Canny': + # pipe = FluxControlPipeline.from_pretrained("black-forest-labs/FLUX.1-Canny-dev", torch_dtype=torch.bfloat16, revision="refs/pr/1").to("cuda") + install('controlnet-aux') + install('timm==0.9.16') + if shared.sd_model.__class__.__name__ != 'FluxControlPipeline' or 'Canny' not in shared.opts.sd_model_checkpoint: + shared.opts.data["sd_model_checkpoint"] = "black-forest-labs/FLUX.1-Canny-dev" + sd_models.reload_model_weights(op='model', revision="refs/pr/1") + if processor_canny is None: + from controlnet_aux import CannyDetector + processor_canny = CannyDetector() + control_image = processor_canny(image, low_threshold=50, high_threshold=200, detect_resolution=1024, image_resolution=1024) + p.task_args['control_image'] = control_image + if strength: + p.task_args['strength'] = None + else: + if processor_canny is not None: + shared.log.debug(f'{title}: tool=Canny unload processor') + processor_canny = None + + if tool == 'Depth': + # pipe = FluxControlPipeline.from_pretrained("black-forest-labs/FLUX.1-Depth-dev", torch_dtype=torch.bfloat16, revision="refs/pr/1").to("cuda") + install('git+https://github.com/asomoza/image_gen_aux.git', 'image_gen_aux') + if shared.sd_model.__class__.__name__ != 'FluxControlPipeline' or 'Depth' not in shared.opts.sd_model_checkpoint: + shared.opts.data["sd_model_checkpoint"] = "black-forest-labs/FLUX.1-Depth-dev" + sd_models.reload_model_weights(op='model', revision="refs/pr/1") + if processor_depth is None: + from image_gen_aux import DepthPreprocessor + processor_depth = DepthPreprocessor.from_pretrained("LiheYoung/depth-anything-large-hf") + control_image = processor_depth(control_image)[0].convert("RGB") + p.task_args['control_image'] = control_image + if strength: + p.task_args['strength'] = None + else: + if processor_depth is not None: + shared.log.debug(f'{title}: tool=Depth unload processor') + processor_depth = None + + shared.log.debug(f'{title}: tool={tool} ready time={time.time() - t0:.2f}') + devices.torch_gc() diff --git a/scripts/image2video.py b/scripts/image2video.py index 876ed3193..5e08922ee 100644 --- a/scripts/image2video.py +++ b/scripts/image2video.py @@ -13,7 +13,7 @@ MODELS = [ class Script(scripts.Script): def title(self): - return 'Video VGen Image-to-Video' + return 'Video: VGen Image-to-Video' def show(self, is_img2img): return is_img2img if shared.native else False diff --git a/wiki b/wiki index 30f3265bb..313a6b911 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 30f3265bb06ac738e4467f58be4df3fc4b49c08b +Subproject commit 313a6b911bd239b4fa8092ed89b936428214342e