From 3e3adcee749d5b804ce2617110085da35421cacb Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 7 Aug 2025 08:44:46 -0400 Subject: [PATCH] add skyreels-v2 Signed-off-by: Vladimir Mandic --- .eslintrc.json | 1 + CHANGELOG.md | 15 +++++--- TODO.md | 3 +- extensions-builtin/sd-extension-system-info | 2 +- html/locale_en.json | 2 +- javascript/contextMenus.js | 2 +- javascript/nvml.js | 4 ++ javascript/sdnext.css | 5 +++ javascript/startup.js | 1 + modules/ui_sections.py | 11 +++--- modules/ui_video.py | 2 +- modules/video_models/models_def.py | 41 +++++++++++++++++++++ modules/video_models/video_load.py | 3 ++ modules/video_models/video_overrides.py | 7 +++- modules/video_models/video_utils.py | 2 +- 15 files changed, 82 insertions(+), 19 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index 7f4c9f27f..38bcae860 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -109,6 +109,7 @@ "jobStatusEl": "readonly", "removeSplash": "readonly", "initNVML": "readonly", + "startNVML": "readonly", "disableNVML": "readonly", "idbGet": "readonly", "idbPut": "readonly", diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f1f03b61..40ae34f9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,18 +4,18 @@ - Qwen with offloading: -## Update for 2025-08-06 +## Update for 2025-08-07 -### Highlights for 2025-08-06 +### Highlights for 2025-08-07 -This time we have several new models: [Qwen-Image](https://qwenlm.github.io/blog/qwen-image/), [FLUX.1-Krea-Dev](https://www.krea.ai/blog/flux-krea-open-source-release) and [Chroma](https://huggingface.co/lodestones/Chroma) -Continuing with major UI work, there is new embedded Docs/Wiki search, redesigned CivitAI integration and quite a few UI updates! +Several new models: [Qwen-Image](https://qwenlm.github.io/blog/qwen-image/), [FLUX.1-Krea-Dev](https://www.krea.ai/blog/flux-krea-open-source-release), [Chroma](https://huggingface.co/lodestones/Chroma), [SkyReels-V2](https://huggingface.co/Skywork/SkyReels-V2-DF-14B-720P-Diffusers) +Plus continuing with major **UI** work, there is new embedded **Docs/Wiki** search, redesigned real-time **hints**, **CivitAI** integration and more! On the compute side, new profiles for high-vram GPUs and offloading improvements And (*as always*) many bugfixes and improvements to existing features! [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) -### Details for 2025-08-06 +### Details for 2025-08-07 - **Models** - [Qwen-Image](https://qwenlm.github.io/blog/qwen-image/) @@ -31,6 +31,11 @@ And (*as always*) many bugfixes and improvements to existing features! great model based on FLUX.1 and then redesigned and retrained by *lodestones* update with latest **v48**, **v48 Detail Calibrated** and **v46 Flash** variants available via *networks -> models -> reference* + - [SkyReels-V2](https://huggingface.co/Skywork/SkyReels-V2-DF-14B-720P-Diffusers) + SkyReels-V2 is a genarative video model based on Wan-2.1 but with heavily modified execution to allow for infinite-length video generation + supported variants are: + - diffusion-forcing: T2I DF 1.3B for 540p videos, T2I DF 14B for 720p videos, I2I DF 14B for 720p videos + - standard: T2I 14B for 720p videos and I2I 14B for 720p videos **Torch** - Set default for ROCm and OpenVINO to `torch==2.8.0` - **UI** diff --git a/TODO.md b/TODO.md index af686fbda..b3b35f3e0 100644 --- a/TODO.md +++ b/TODO.md @@ -30,8 +30,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - [IPAdapter composition](https://huggingface.co/ostris/ip-composition-adapter) - [STG](https://github.com/huggingface/diffusers/blob/main/examples/community/README.md#spatiotemporal-skip-guidance) - [SmoothCache](https://github.com/huggingface/diffusers/issues/11135) -- [MagCache](https://github.com/lllyasviel/FramePack/pull/673/files) -- [HiDream GGUF](https://github.com/huggingface/diffusers/pull/11550) +- [MagCache](https://github.com/lllyasviel/FramePack/pull/673/files) - [Nunchaku PulID](https://github.com/mit-han-lab/nunchaku/pull/274) - [Dream0 guidance](https://huggingface.co/ByteDance/DreamO) - [SUPIR upscaler](https://github.com/Fanghua-Yu/SUPIR) diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info index 0760f3bce..615d2f810 160000 --- a/extensions-builtin/sd-extension-system-info +++ b/extensions-builtin/sd-extension-system-info @@ -1 +1 @@ -Subproject commit 0760f3bcee4cd8448089e749dd7c22cdfebf15c3 +Subproject commit 615d2f8103fb61779037bd81523906457fbf7277 diff --git a/html/locale_en.json b/html/locale_en.json index e908743eb..d35d8a96d 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -93,7 +93,7 @@ {"id":"","label":"Denoise","localized":"","hint":"Denoising settings. Higher denoise means that more of existing image content is allowed to change during generate"}, {"id":"","label":"Mask","localized":"","hint":"Image masking and mask options"}, {"id":"","label":"Input","localized":"","hint":"Selection of input media"}, - {"id":"","label":"Video","localized":"","hint":"Settings related to video generation"}, + {"id":"","label":"Video","localized":"","hint":"Create video using guidance"}, {"id":"","label":"Control elements","localized":"","hint":"Control elements are advanced models that can guide generation towards desired outcome"}, {"id":"","label":"IP adapter","localized":"","hint":"Guide generation towards desired outcome using IP adapters plugin models"}, {"id":"","label":"IP adapters","localized":"","hint":"IP adapters are plugin models that can guide generation towards desired outcome"}, diff --git a/javascript/contextMenus.js b/javascript/contextMenus.js index 271f5a9e4..3dfa16b7e 100644 --- a/javascript/contextMenus.js +++ b/javascript/contextMenus.js @@ -149,7 +149,7 @@ async function initContextMenu() { appendContextMenuOption(id, 'Generate forever', () => generateForever(`#${tab}_generate`)); appendContextMenuOption(id, 'Apply selected style', quickApplyStyle); appendContextMenuOption(id, 'Quick save style', quickSaveStyle); - appendContextMenuOption(id, 'nVidia overlay', initNVML); + appendContextMenuOption(id, 'nVidia overlay', startNVML); id = `#${tab}_reprocess`; appendContextMenuOption(id, 'Decode full quality', () => reprocessClick(`${tab}`, 'reprocess_decode'), true); appendContextMenuOption(id, 'Refine & HiRes pass', () => reprocessClick(`${tab}`, 'reprocess_refine'), true); diff --git a/javascript/nvml.js b/javascript/nvml.js index cf55bb196..155dc3f89 100644 --- a/javascript/nvml.js +++ b/javascript/nvml.js @@ -90,6 +90,10 @@ async function initNVML() { gradioApp().appendChild(nvmlEl); log('initNVML'); } +} + +async function startNVML() { + nvmlEl = document.getElementById('nvml'); if (nvmlInterval) { clearInterval(nvmlInterval); nvmlInterval = null; diff --git a/javascript/sdnext.css b/javascript/sdnext.css index f5e2588e9..636aeaec8 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -2055,6 +2055,11 @@ div:has(>#tab-gallery-folders) { font-weight: bold; } +.video-model-link { + color: var(--button-primary-background-fill); + font-weight: normal; +} + @keyframes move { from { background-position-x: 0, -40px; diff --git a/javascript/startup.js b/javascript/startup.js index 842f925ba..3629511ca 100644 --- a/javascript/startup.js +++ b/javascript/startup.js @@ -39,6 +39,7 @@ async function initStartup() { executeCallbacks(uiReadyCallbacks); initLogMonitor(); setupExtraNetworks(); + initNVML(); // optinally wait for modern ui if (window.waitForUiReady) await waitForUiReady(); diff --git a/modules/ui_sections.py b/modules/ui_sections.py index 94fbde348..4f72783bb 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -134,12 +134,13 @@ def create_video_inputs(tab:str, show_always:bool=False): gr.update(visible=video_type not in ['None', 'GIF', 'PNG'] or show_always), gr.update(visible=video_type not in ['None', 'GIF', 'PNG'] or show_always), ] - with gr.Column(): + with gr.Row(): video_codecs = ['None', 'GIF', 'PNG', 'MP4/MP4V', 'MP4/AVC1', 'MP4/JVT3', 'MKV/H264', 'AVI/DIVX', 'AVI/RGBA', 'MJPEG/MJPG', 'MPG/MPG1', 'AVR/AVR1'] - video_type = gr.Dropdown(label='Save video', choices=video_codecs, value='None', elem_id=f"{tab}_video_type") - with gr.Column(): - video_duration = gr.Slider(label='Duration', minimum=0.25, maximum=300, step=0.25, value=2, visible=show_always, elem_id=f"{tab}_video_duration") - video_loop = gr.Checkbox(label='Loop', value=True, visible=show_always, elem_id=f"{tab}_video_loop") + video_type = gr.Dropdown(label='Video format', choices=video_codecs, value='MP4/MP4V', elem_id=f"{tab}_video_type") + with gr.Row(): + video_duration = gr.Slider(label='Video duration', minimum=0.25, maximum=300, step=0.25, value=2, visible=show_always, elem_id=f"{tab}_video_duration") + video_loop = gr.Checkbox(label='Loop video', value=True, visible=show_always, elem_id=f"{tab}_video_loop") + with gr.Row(): video_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=show_always, elem_id=f"{tab}_video_pad") video_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=show_always, elem_id=f"{tab}_video_interpolate") video_type.change(fn=video_type_change, inputs=[video_type], outputs=[video_duration, video_loop, video_pad, video_interpolate]) diff --git a/modules/ui_video.py b/modules/ui_video.py index e12f08bc9..bb9cace94 100644 --- a/modules/ui_video.py +++ b/modules/ui_video.py @@ -27,7 +27,7 @@ def create_ui(): with gr.Row(elem_id="video_interface", equal_height=False): with gr.Tabs(elem_classes=['video-tabs'], elem_id='video-tabs'): overrides = ui_common.create_override_inputs('video') - with gr.Tab('Generic', id='video-tab') as video_tab: + with gr.Tab('Core', id='video-tab') as video_tab: from modules.video_models import video_ui video_ui.create_ui(prompt, negative, styles, overrides) with gr.Tab('FramePack', id='framepack-tab') as framepack_tab: diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index e6f570775..4111ee517 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -9,13 +9,16 @@ class Model(): url: str = '' repo: str = None repo_cls: classmethod = None + repo_revision: str = None dit: str = None dit_cls: classmethod = None dit_folder: str = 'transformer' + dit_revision: str = None te: str = None te_cls: classmethod = None te_folder: str = 'text_encoder' te_hijack: bool = True + te_revision: str = None image_hijack: bool = True vae_hijack: bool = True vae_remote: bool = False @@ -195,6 +198,44 @@ models = { te_cls=transformers.T5EncoderModel, dit_cls=diffusers.WanTransformer3DModel), ], + 'SkyReels V2': [ + Model(name='None'), + Model(name='SkyReels-V2 T2I-DF 1.3B-540P', + url='https://huggingface.co/Skywork/SkyReels-V2-DF-1.3B-540P-Diffusers', + repo='Skywork/SkyReels-V2-DF-1.3B-540P-Diffusers', + repo_cls=diffusers.SkyReelsV2DiffusionForcingPipeline, + repo_revision='refs/pr/1', + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.SkyReelsV2Transformer3DModel), + Model(name='SkyReels-V2 T2I-DF 14B-720P', + url='https://huggingface.co/Skywork/SkyReels-V2-DF-14B-720P-Diffusers', + repo='Skywork/SkyReels-V2-DF-14B-720P-Diffusers', + repo_cls=diffusers.SkyReelsV2DiffusionForcingPipeline, + repo_revision='refs/pr/1', + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.SkyReelsV2Transformer3DModel), + Model(name='SkyReels-V2 I2I-DF 14B-720P', + url='https://huggingface.co/Skywork/SkyReels-V2-DF-14B-720P-Diffusers', + repo='Skywork/SkyReels-V2-DF-14B-720P-Diffusers', + repo_cls=diffusers.SkyReelsV2DiffusionForcingImageToVideoPipeline, + repo_revision='refs/pr/1', + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.SkyReelsV2Transformer3DModel), + Model(name='SkyReels-V2 T2I 14B-720P', + url='https://huggingface.co/Skywork/SkyReels-V2-T2V-14B-720P-Diffusers', + repo='Skywork/SkyReels-V2-T2V-14B-720P-Diffusers', + repo_cls=diffusers.SkyReelsV2Pipeline, + repo_revision='refs/pr/1', + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.SkyReelsV2Transformer3DModel), + Model(name='SkyReels-V2 I2I 14B-720P', + url='https://huggingface.co/Skywork/SkyReels-V2-I2V-14B-720P-Diffusers', + repo='Skywork/SkyReels-V2-I2V-14B-720P-Diffusers', + repo_cls=diffusers.SkyReelsV2ImageToVideoPipeline, + repo_revision='refs/pr/1', + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.SkyReelsV2Transformer3DModel), + ], 'Mochi Video': [ Model(name='None'), Model(name='Mochi 1 T2V', diff --git a/modules/video_models/video_load.py b/modules/video_models/video_load.py index 3eda8f735..53b4d66a0 100644 --- a/modules/video_models/video_load.py +++ b/modules/video_models/video_load.py @@ -28,6 +28,7 @@ def load_model(selected: models_def.Model): text_encoder = selected.te_cls.from_pretrained( pretrained_model_name_or_path=selected.te or selected.repo, subfolder=selected.te_folder, + revision=selected.te_revision or selected.repo_revision, cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args @@ -44,6 +45,7 @@ def load_model(selected: models_def.Model): transformer = selected.dit_cls.from_pretrained( pretrained_model_name_or_path=selected.dit or selected.repo, subfolder=selected.dit_folder, + revision=selected.dit_revision or selected.repo_revision, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, **quant_args @@ -63,6 +65,7 @@ def load_model(selected: models_def.Model): pretrained_model_name_or_path=selected.repo, transformer=transformer, text_encoder=text_encoder, + revision=selected.repo_revision, cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **kwargs, diff --git a/modules/video_models/video_overrides.py b/modules/video_models/video_overrides.py index 655256e95..07cf1cd23 100644 --- a/modules/video_models/video_overrides.py +++ b/modules/video_models/video_overrides.py @@ -10,10 +10,13 @@ debug = shared.log.trace if os.environ.get('SD_VIDEO_DEBUG', None) is not None e def load_override(selected: Model): kwargs = {} - if selected.name == 'Allegro T2V': + # Allegro + if 'Allegro T2V' in selected.name: kwargs['vae'] = diffusers.AutoencoderKLAllegro.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir) - if selected.name == 'LTXVideo 0.9.5 I2V': + # LTX + if 'LTXVideo 0.9.5 I2V' in selected.name: kwargs['vae'] = diffusers.AutoencoderKLLTXVideo.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir) + # WAN if 'WAN 2.1 14B' in selected.name: kwargs['vae'] = diffusers.AutoencoderKLWan.from_pretrained(selected.repo, subfolder="vae", torch_dtype=torch.float32, cache_dir=shared.opts.hfcache_dir) debug(f'Video overrides: model="{selected.name}" kwargs={list(kwargs)}') diff --git a/modules/video_models/video_utils.py b/modules/video_models/video_utils.py index cbca9a3f5..47faeac13 100644 --- a/modules/video_models/video_utils.py +++ b/modules/video_models/video_utils.py @@ -14,7 +14,7 @@ def queue_err(msg): def get_url(url): - return f'  {url}

' if url else '

' + return f'{url}

' if url else '

' def check_av():