diff --git a/modules/dml/hijack/tomesd.py b/modules/dml/hijack/tomesd.py index ff7de1a38..79de721df 100644 --- a/modules/dml/hijack/tomesd.py +++ b/modules/dml/hijack/tomesd.py @@ -1,8 +1,8 @@ from typing import Type import torch -import tomesd from modules.dml.hijack.utils import catch_nan + def make_tome_block(block_class: Type[torch.nn.Module]) -> Type[torch.nn.Module]: class ToMeBlock(block_class): # Save for unpatching later @@ -19,4 +19,9 @@ def make_tome_block(block_class: Type[torch.nn.Module]) -> Type[torch.nn.Module] return x return ToMeBlock -tomesd.patch.make_tome_block = make_tome_block + +try: + import tomesd + tomesd.patch.make_tome_block = make_tome_block +except Exception: + pass diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py index 2445e24ff..93b5d3dd1 100644 --- a/modules/generation_parameters_copypaste.py +++ b/modules/generation_parameters_copypaste.py @@ -247,8 +247,8 @@ def parse_generation_parameters(infotext, no_prompt=False): else: params[k] = v if not no_prompt: - params["Prompt"] = prompt.replace('Prompt:', '').strip(' ,') - params["Negative prompt"] = negative.replace('Negative prompt:', '').strip(' ,') + params["Prompt"] = prompt.replace('Prompt:', '').strip(' ,\n') + params["Negative prompt"] = negative.replace('Negative prompt:', '').strip(' ,\n') debug(f"Parse: {params}") return params diff --git a/modules/sd_models.py b/modules/sd_models.py index 1dd6a6abb..e70ff4a47 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1620,6 +1620,8 @@ def apply_token_merging(sd_model): shared.log.warning('Token merging not supported with HyperTile for UNet') return try: + import installer + installer.install('tomesd', 'tomesd', ignore=False) import tomesd tomesd.apply_patch( sd_model, diff --git a/requirements.txt b/requirements.txt index 200f43a19..7fdd72093 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,8 +57,7 @@ pandas protobuf==3.20.3 pytorch_lightning==1.9.4 tokenizers==0.19.1 -transformers==4.40.1 -tomesd==0.1.3 +transformers==4.40.2 urllib3==1.26.18 Pillow==10.3.0 timm==0.9.16 diff --git a/scripts/mulan.py b/scripts/mulan.py new file mode 100644 index 000000000..5c658f450 --- /dev/null +++ b/scripts/mulan.py @@ -0,0 +1,138 @@ +# https://github.com/mulanai/MuLan +# https://huggingface.co/mulanai/mulan-lang-adapter +# https://huggingface.co/OpenGVLab/InternVL-14B-224px + +""" +- [MuLan](https://github.com/mulanai/MuLan) Multi-langunage prompts - wirte your prompts in ~110 auto-detected languages! + Compatible with SD15 and SDXL + Enable in scripts -> MuLan and set encoder to `InternVL-14B-224px` encoder + (that is currently only supported encoder, but others will be added) + Note: Model will be auto-downloaded on first use: note its huge size of 27GB + Even executing it in FP16 context will require ~16GB of VRAM for text encoder alone + *Note*: Uses fixed prompt parser, so no prompt attention will be used + +Examples: +- English: photo of a beautiful woman wearing a white bikini on a beach with a city skyline in the background +- Croatian: fotografija lijepe žene u bijelom bikiniju na plaži s gradskim obzorom u pozadini +- Italian: Foto di una bella donna che indossa un bikini bianco su una spiaggia con lo skyline di una città sullo sfondo +- Spanish: Foto de una hermosa mujer con un bikini blanco en una playa con un horizonte de la ciudad en el fondo +- German: Foto einer schönen Frau in einem weißen Bikini an einem Strand mit einer Skyline der Stadt im Hintergrund +- Arabic: صورة لامرأة جميلة ترتدي بيكيني أبيض على شاطئ مع أفق المدينة في الخلفية +- Japanese: 街のスカイラインを背景にビーチで白いビキニを着た美しい女性の写真 +- Chinese: 一个美丽的女人在海滩上穿着白色比基尼的照片,背景是城市天际线 +- Korean: 도시의 스카이라인을 배경으로 해변에서 흰색 비키니를 입은 아름 다운 여성의 사진 +""" + +import gradio as gr +from modules import shared, scripts, processing, devices + + +ENCODERS =[ + # 'None', + 'OpenGVLab/InternVL-14B-224px', + # 'OpenGVLab/InternViT-6B-224px', + # 'OpenGVLab/InternViT-6B-448px-V1-0', + # 'OpenGVLab/InternViT-6B-448px-V1-2', + # 'OpenGVLab/InternViT-6B-448px-V1-5', +] +GITPATH = 'git+https://github.com/mulanai/MuLan' + +pipe_type = None +adapter = None +text_encoder = None +tokenizer = None +text_encoder_path = None + + +class Script(scripts.Script): + def title(self): + return 'MuLan' + + def show(self, is_img2img): + if shared.cmd_opts.experimental: + return True if shared.backend == shared.Backend.DIFFUSERS else False + else: + return False + + def ui(self, _is_img2img): + with gr.Row(): + gr.HTML('  MuLan
') + with gr.Row(): + selected_encoder = gr.Dropdown(label='Encoder', choices=ENCODERS, value=ENCODERS[0]) + return [selected_encoder] + + def run(self, p: processing.StableDiffusionProcessing, selected_encoder): # pylint: disable=arguments-differ + global pipe_type, adapter, text_encoder, tokenizer, text_encoder_path # pylint: disable=global-statement + if not selected_encoder or selected_encoder == 'None': + return + # create pipeline + if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl': + shared.log.error(f'MuLan: incorrect base model: {shared.sd_model.__class__.__name__}') + return + + adapter_path = None + if shared.sd_model_type == 'sd': + adapter_path = 'mulanai/mulan-lang-adapter::sd15_aesthetic.pth' + if shared.sd_model_type == 'sdxl': + adapter_path = 'mulanai/mulan-lang-adapter::sdxl_aesthetic.pth' + if adapter_path is None: + return + + # install-on-demand + import installer + installer.install(GITPATH, 'mulankit') + import mulankit + + # backup pipeline and params + orig_pipeline = shared.sd_model + orig_prompt_attention = shared.opts.prompt_attention + + # mulan only works with single image, single prompt and in fixed attention + p.batch_size = 1 + p.n_iter = 1 + shared.opts.prompt_attention = 'Fixed attention' + if isinstance(p.prompt, list): + p.prompt = p.prompt[0] + p.task_args['prompt'] = p.prompt + if isinstance(p.negative_prompt, list): + p.prompt = p.negative_prompt[0] + p.task_args['negative_prompt'] = p.negative_prompt + + if pipe_type != ('sd15' if shared.sd_model_type == 'sd' else 'sdxl'): + pipe_type = 'sd15' if shared.sd_model_type == 'sd' else 'sdxl' + adapter = None + if text_encoder is None or tokenizer is None or text_encoder_path != selected_encoder: + text_encoder_path = selected_encoder + shared.log.debug(f'MuLan loading: encoder="{text_encoder_path}"') + text_encoder = None + tokenizer = None + devices.torch_gc(force=True) + text_encoder, tokenizer = mulankit.api.load_internvl(text_encoder_path, text_encoder, tokenizer, torch_dtype=shared.sd_model.text_encoder.dtype) + devices.torch_gc(force=True) + if adapter is None: + shared.log.debug(f'MuLan loading: adapter="{adapter_path}"') + adapter = None + devices.torch_gc(force=True) + adapter = mulankit.api.load_adapter(adapter_path, type=pipe_type) + devices.torch_gc(force=True) + + if not getattr(shared.sd_model, 'mulan', False): + shared.log.info(f'MuLan apply: adapter="{adapter_path}" encoder="{text_encoder_path}"') + # mulankit.setup(force_sdxl_zero_empty_prompt=False, force_sdxl_zero_pool_prompt=False) + shared.sd_model = mulankit.transform(shared.sd_model, + adapter=adapter, + adapter_path=adapter_path, + text_encoder=text_encoder, + text_encoder_path=text_encoder_path, + pipe_type=pipe_type, + replace=False) + shared.sd_model.mulan = True + devices.torch_gc(force=True) + + processing.fix_seed(p) + processed: processing.Processed = processing.process_images(p) # runs processing using main loop + + # restore pipeline and params + shared.opts.data['prompt_attention'] = orig_prompt_attention + shared.sd_model = orig_pipeline + return processed