From 82e9745c65e9d4074ee2f1414d6aaeb88b3a7e2a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 8 May 2026 14:44:00 +0200 Subject: [PATCH] add joyai-image-edit Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 29 +++++++---- data/reference.json | 12 ++++- extensions-builtin/sdnext-modernui | 2 +- ...opensource--JoyAI-Image-Edit-Diffusers.jpg | 0 modules/modeldata.py | 2 + modules/progress.py | 4 +- modules/sd_detect.py | 2 + modules/sd_models.py | 14 ++++++ modules/shared_items.py | 3 ++ pipelines/model_joy.py | 50 +++++++++++++++++++ 10 files changed, 103 insertions(+), 15 deletions(-) create mode 100644 models/Reference/jdopensource--JoyAI-Image-Edit-Diffusers.jpg create mode 100644 pipelines/model_joy.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9700fe346..40dc4aad5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ *What's New?* - Image editing models now can work with multiple image inputs! -- New models: *Step1X-Edit*, *VIBE Image Edit* and *UltraFlux* plus enhanced capabilities for *Anima*, *Ernie-Image*, *LTX* and *Chroma* models +- New models: *JoyAI Image Edit*, *Step1X-Edit*, *VIBE Image Edit* and *UltraFlux* plus enhanced capabilities for *Anima*, *Ernie-Image*, *LTX* and *Chroma* models - UI improvements accross the board: *Main panels*, *Gallery*, *Kanvas*, and more For full details, see [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) @@ -16,7 +16,7 @@ For full details, see [ChangeLog](https://github.com/vladmandic/automatic/blob/m ### Details for 2026-05-08 - **Features** - - **Multi-image** workflows! + - **Multi-image** workflows! for models that support multiple images as inputs, you can now add multiple stages in Kanvas prompts like "*place character from first image, add background from second image, render in style from third image*" are now possible - option *inputs -> skip processing* to force images to passed to model as-is without any pre-processing @@ -32,26 +32,32 @@ For full details, see [ChangeLog](https://github.com/vladmandic/automatic/blob/m still a popular method for upscaling, but has not been updated nor maintained for a while so now its modernized and fully integrated as a built-in script! - **Models** - - [StepFun Step1X-Edit v1.1](https://huggingface.co/stepfun-ai/Step1X-Edit-v1p1-diffusers) image edit model support + - [JoyAI Image Edit](https://huggingface.co/jdopensource/JoyAI-Image-Edit-Diffusers) image-editing model support + includes multimodal conditioning using *Qwen3-VL* with a dedicated *JoyImageEdit* diffusion transformer + *note* this is a large model at 50GB so use of agressive quantization is recommended + - [StepFun Step1X-Edit v1.1](https://huggingface.co/stepfun-ai/Step1X-Edit-v1p1-diffusers) image-editing model support step1x is a large dedicated image edit model combining qwen-2.5 8B encoder with custom 12.4B transformer - - [VIBE Image Edit](https://huggingface.co/iitolstykh/VIBE-Image-Edit) text-guided image editing model + - [VIBE Image Edit](https://huggingface.co/iitolstykh/VIBE-Image-Edit) image-editing model support built on Sana1.5-1.6B diffusion backbone with Qwen3-VL-2B multimodal conditioning - supports both *T2I* and *I2I* workflows, uses multi-scale resolution binning up to 2048px + primarily image-editing model, but supports t2i as well, uses multi-scale resolution binning up to 2048px - [AlphaVLLM Lumina-DiMOO](https://huggingface.co/Alpha-VLLM/Lumina-DiMOO) unified multimodal diffusion model includes *T2I*, *I2I edit*, and *MMU* capabilities in a single pipeline - *note* in addition to normal prompt-based image editing, model also supports special prompts: *dense, canny_pred, control, subject, edit, ref_transfer, multi_view* - *note* as with most multi-modal/unified models, it needs higher step count (recommended is 64 steps) and uses quite a lot of VRAM, so use with caution! - - [Owen777 UltraFlux-v1](https://huggingface.co/Owen777/UltraFlux-v1) native 4K text-to-image model based on FLUX.1-dev - *note*: UltraFlux is capable of rendering images up to 4K resolution, but it doesnt mean it will do that on any hardware - it will depend on your VRAM! + *note* in addition to normal prompt-based image editing, model also supports special prompts: *dense, canny_pred, control, subject, edit, ref_transfer, multi_view* + *note* as with most multi-modal/unified models, it needs higher step count (recommended is 64 steps) and uses quite a lot of VRAM, so use with caution! + - [Owen777 UltraFlux-v1](https://huggingface.co/Owen777/UltraFlux-v1) native 4K text-to-image model based on *FLUX.1-dev* + *note*: UltraFlux is capable of rendering images up to 4K resolution, but it doesnt mean it will do that on any hardware - it will depend on your VRAM! - **UI** - - add button to manually reorient input/output panels - - all ui panels can be minimized/maximized by clicking on their header + - **Networks** using networks to load model or auto-download a reference model will now be reflected in the UI + - ability to manually reorient *input/output* panels + - all ui panels can be *minimized/maximized* by clicking on their header state is preserved across sessions and can be used to hide rarely used panels and declutter the workspace - **Kanvas** re-order stages by clicking on active stage order of stages detemines order of images passed to model - **Kanvas** *magic-wand* tool now works on mask layer and auto-creates mask based on perceptual tolerance - **Gallery** add thumbnail size slider - **Gallery** add quick info/download/delete buttons on thumbnail hover + - **Models** sortable columns, ability to remove a model + applies to models as well as huggingface cache entries - **Control** - remove buttons: *input/control/process* - move params *control input type* to control menu section @@ -77,6 +83,7 @@ For full details, see [ChangeLog](https://github.com/vladmandic/automatic/blob/m - `ipex` invalid device type - cache network thumbnails - `scripts` corrupting control ui state + - avoid `callback` duplicate registrations ## Update for 2026-04-28 diff --git a/data/reference.json b/data/reference.json index 0e21aaf5c..7e46786ff 100644 --- a/data/reference.json +++ b/data/reference.json @@ -961,8 +961,18 @@ "desc": "VIBE is an open-source text-guided image editing model combining Sana1.5-1.6B diffusion backbone with Qwen3-VL multimodal conditioning for fast, instruction-based edits.", "skip": true, "extras": "sampler: Default, cfg_scale: 4.5, image_guidance_scale: 1.2, steps: 20", - "size": 9.72, + "size": 9.27, "date": "2025 December" + }, + + "JoyAI Image Edit": { + "path": "jdopensource/JoyAI-Image-Edit-Diffusers", + "preview": "jdopensource--JoyAI-Image-Edit-Diffusers.jpg", + "desc": "JoyAI Image Edit is a Diffusers-native image editing model that combines a JoyImageEdit transformer with Qwen3-VL multimodal conditioning for instruction-guided edits.", + "skip": true, + "size": 50.31, + "extras": "sampler: Default", + "date": "2026 April" } } diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 74fd68c5f..4f23bc8b9 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 74fd68c5f09716a6a3d0df5ed40780262774fa93 +Subproject commit 4f23bc8b969a41fb490b38d49b59a8fb25a2048f diff --git a/models/Reference/jdopensource--JoyAI-Image-Edit-Diffusers.jpg b/models/Reference/jdopensource--JoyAI-Image-Edit-Diffusers.jpg new file mode 100644 index 000000000..e69de29bb diff --git a/modules/modeldata.py b/modules/modeldata.py index bdd61ddd6..1c2acf2d7 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -110,6 +110,8 @@ def get_model_type(pipe): model_type = 'glmimage' elif 'Step1XEdit' in name: model_type = 'step1x_edit' + elif 'JoyImageEdit' in name: + model_type = 'joy' elif 'OvisImage' in name or 'Ovis-Image' in name: model_type = 'ovis' elif 'Wan' in name: diff --git a/modules/progress.py b/modules/progress.py index 05f50077e..2fcd4dc22 100644 --- a/modules/progress.py +++ b/modules/progress.py @@ -69,7 +69,7 @@ class InternalProgressResponse(BaseModel): def api_progress(req: ProgressRequest): - active = req.id_task == current_task + active = (req.id_task == current_task) or (req.id_task == 'undefined' and len(shared.state.job) > 0) queued = req.id_task in pending_tasks completed = req.id_task in finished_tasks paused = shared.state.paused @@ -92,7 +92,7 @@ def api_progress(req: ProgressRequest): id_live_preview = -1 textinfo = "Queued..." if queued else "Waiting..." - debug_log(f'Preview: job={shared.state.job} active={active} progress={step}/{steps}/{progress} image={shared.state.current_image_sampling_step} request={id_live_preview} last={shared.state.id_live_preview} job={shared.state.preview_job} elapsed={elapsed:.3f}') + debug_log(f'Progress: job="{shared.state.job}" active={active} progress={step}/{steps}/{progress} image={shared.state.current_image_sampling_step} request={id_live_preview} last={shared.state.id_live_preview} job={shared.state.preview_job} elapsed={elapsed:.3f}') if active and (req.id_live_preview != -1): have_image = shared.state.set_current_image() diff --git a/modules/sd_detect.py b/modules/sd_detect.py index a37716977..12acc4f85 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -162,6 +162,8 @@ def guess_by_name(fn, current_guess): new_guess = 'Step1X-Edit' elif 'vibe-image-edit' in fn.lower(): new_guess = 'VIBE' + elif 'joyai-image-edit' in fn.lower() or 'joy-image-edit' in fn.lower(): + new_guess = 'Joy' if debug_load: log.trace(f'Autodetect: method=name file="{fn}" previous="{current_guess}" current="{new_guess}"') return new_guess or current_guess diff --git a/modules/sd_models.py b/modules/sd_models.py index a0806b165..50898acfb 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -469,6 +469,10 @@ def load_diffuser_force(detected_model_type, checkpoint_info, diffusers_load_con from pipelines.model_vibe import load_vibe sd_model = load_vibe(checkpoint_info, diffusers_load_config) allow_post_quant = False + elif model_type in ['Joy']: + from pipelines.model_joy import load_joy + sd_model = load_joy(checkpoint_info, diffusers_load_config) + allow_post_quant = False elif model_type in ['Qwen']: from pipelines.model_qwen import load_qwen sd_model = load_qwen(checkpoint_info, diffusers_load_config) @@ -1489,3 +1493,13 @@ def save_model(name: str, path: str | None = None, shard: str = "5GB", overwrite log.error(f'Save model: path="{model_name}" {e}') errors.display(e, 'Save model') return f'Error: {e}' + + +def list_hfcache(): + checkpoints = [] + for f in os.scandir(shared.opts.hfcache_dir): + if not os.path.isdir(f) or not f.name.startswith('models--'): + continue + checkpoint = CheckpointInfo(filename=f.path, name=path_to_repo(f.name), model_type='hfcache') + checkpoints.append(checkpoint) + return checkpoints diff --git a/modules/shared_items.py b/modules/shared_items.py index b455483ba..8cdd3d396 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -12,6 +12,7 @@ pipelines = { 'Stable Diffusion Inpaint': getattr(diffusers, 'StableDiffusionInpaintPipeline', None), 'Stable Diffusion Instruct': getattr(diffusers, 'StableDiffusionInstructPix2PixPipeline', None), 'Stable Diffusion 1.5': getattr(diffusers, 'StableDiffusionPipeline', None), + 'Stable Diffusion 2': getattr(diffusers, 'StableDiffusionPipeline', None), 'Stable Diffusion 2.x': getattr(diffusers, 'StableDiffusionPipeline', None), 'Stable Diffusion Upscale': getattr(diffusers, 'StableDiffusionUpscalePipeline', None), 'Stable Diffusion XL': getattr(diffusers, 'StableDiffusionXLPipeline', None), @@ -36,6 +37,7 @@ pipelines = { 'Kandinsky 2.1': getattr(diffusers, 'KandinskyCombinedPipeline', None), 'Kandinsky 2.2': getattr(diffusers, 'KandinskyV22CombinedPipeline', None), 'Kandinsky 3.0': getattr(diffusers, 'Kandinsky3Pipeline', None), + 'Kandinsky 5.0': getattr(diffusers, 'Kandinsky5T2IPipeline', None), 'Wuerstchen': getattr(diffusers, 'WuerstchenCombinedPipeline', None), 'Kolors': getattr(diffusers, 'KolorsPipeline', None), 'CogView 3': getattr(diffusers, 'CogView3PlusPipeline', None), @@ -47,6 +49,7 @@ pipelines = { 'Cosmos': getattr(diffusers, 'Cosmos2TextToImagePipeline', None), 'WanAI': getattr(diffusers, 'WanPipeline', None), 'Qwen': getattr(diffusers, 'QwenImagePipeline', None), + 'Joy': getattr(diffusers, 'JoyImageEditPipeline', None), 'HunyuanImage': getattr(diffusers, 'HunyuanImagePipeline', None), 'ERNIE-Image': getattr(diffusers, 'ErnieImagePipeline', None), 'Nucleus-Image': getattr(diffusers, 'NucleusMoEImagePipeline', None), diff --git a/pipelines/model_joy.py b/pipelines/model_joy.py new file mode 100644 index 000000000..5ed363315 --- /dev/null +++ b/pipelines/model_joy.py @@ -0,0 +1,50 @@ +import diffusers +import transformers +from modules import shared, devices, sd_models, model_quant, sd_hijack_te, sd_hijack_vae +from modules.logger import log +from pipelines import generic + + +def load_joy(checkpoint_info, diffusers_load_config=None): + if diffusers_load_config is None: + diffusers_load_config = {} + repo_id = sd_models.path_to_repo(checkpoint_info) + sd_models.hf_auth_check(checkpoint_info) + + load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, allow_quant=False) + log.debug(f'Load model: type=JoyImageEdit repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}') + + transformer = generic.load_transformer( + repo_id, + cls_name=diffusers.JoyImageEditTransformer3DModel, + load_config=diffusers_load_config, + ) + text_encoder = generic.load_text_encoder( + repo_id, + cls_name=transformers.Qwen3VLForConditionalGeneration, + load_config=diffusers_load_config, + ) + + pipe = diffusers.JoyImageEditPipeline.from_pretrained( + repo_id, + cache_dir=shared.opts.diffusers_dir, + transformer=transformer, + text_encoder=text_encoder, + **load_args, + ) + pipe.task_args = { + 'output_type': 'np', + } + + diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING['joy-image-edit'] = diffusers.JoyImageEditPipeline + diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING['joy-image-edit'] = diffusers.JoyImageEditPipeline + + generic.load_vae_override(pipe, diffusers_load_config) + + del transformer + del text_encoder + sd_hijack_te.init_hijack(pipe) + sd_hijack_vae.init_hijack(pipe) + + devices.torch_gc(force=True, reason='load') + return pipe