diff --git a/modules/dml/hijack/tomesd.py b/modules/dml/hijack/tomesd.py
index ff7de1a38..79de721df 100644
--- a/modules/dml/hijack/tomesd.py
+++ b/modules/dml/hijack/tomesd.py
@@ -1,8 +1,8 @@
from typing import Type
import torch
-import tomesd
from modules.dml.hijack.utils import catch_nan
+
def make_tome_block(block_class: Type[torch.nn.Module]) -> Type[torch.nn.Module]:
class ToMeBlock(block_class):
# Save for unpatching later
@@ -19,4 +19,9 @@ def make_tome_block(block_class: Type[torch.nn.Module]) -> Type[torch.nn.Module]
return x
return ToMeBlock
-tomesd.patch.make_tome_block = make_tome_block
+
+try:
+ import tomesd
+ tomesd.patch.make_tome_block = make_tome_block
+except Exception:
+ pass
diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py
index 2445e24ff..93b5d3dd1 100644
--- a/modules/generation_parameters_copypaste.py
+++ b/modules/generation_parameters_copypaste.py
@@ -247,8 +247,8 @@ def parse_generation_parameters(infotext, no_prompt=False):
else:
params[k] = v
if not no_prompt:
- params["Prompt"] = prompt.replace('Prompt:', '').strip(' ,')
- params["Negative prompt"] = negative.replace('Negative prompt:', '').strip(' ,')
+ params["Prompt"] = prompt.replace('Prompt:', '').strip(' ,\n')
+ params["Negative prompt"] = negative.replace('Negative prompt:', '').strip(' ,\n')
debug(f"Parse: {params}")
return params
diff --git a/modules/sd_models.py b/modules/sd_models.py
index 1dd6a6abb..e70ff4a47 100644
--- a/modules/sd_models.py
+++ b/modules/sd_models.py
@@ -1620,6 +1620,8 @@ def apply_token_merging(sd_model):
shared.log.warning('Token merging not supported with HyperTile for UNet')
return
try:
+ import installer
+ installer.install('tomesd', 'tomesd', ignore=False)
import tomesd
tomesd.apply_patch(
sd_model,
diff --git a/requirements.txt b/requirements.txt
index 200f43a19..7fdd72093 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -57,8 +57,7 @@ pandas
protobuf==3.20.3
pytorch_lightning==1.9.4
tokenizers==0.19.1
-transformers==4.40.1
-tomesd==0.1.3
+transformers==4.40.2
urllib3==1.26.18
Pillow==10.3.0
timm==0.9.16
diff --git a/scripts/mulan.py b/scripts/mulan.py
new file mode 100644
index 000000000..5c658f450
--- /dev/null
+++ b/scripts/mulan.py
@@ -0,0 +1,138 @@
+# https://github.com/mulanai/MuLan
+# https://huggingface.co/mulanai/mulan-lang-adapter
+# https://huggingface.co/OpenGVLab/InternVL-14B-224px
+
+"""
+- [MuLan](https://github.com/mulanai/MuLan) Multi-langunage prompts - wirte your prompts in ~110 auto-detected languages!
+ Compatible with SD15 and SDXL
+ Enable in scripts -> MuLan and set encoder to `InternVL-14B-224px` encoder
+ (that is currently only supported encoder, but others will be added)
+ Note: Model will be auto-downloaded on first use: note its huge size of 27GB
+ Even executing it in FP16 context will require ~16GB of VRAM for text encoder alone
+ *Note*: Uses fixed prompt parser, so no prompt attention will be used
+
+Examples:
+- English: photo of a beautiful woman wearing a white bikini on a beach with a city skyline in the background
+- Croatian: fotografija lijepe žene u bijelom bikiniju na plaži s gradskim obzorom u pozadini
+- Italian: Foto di una bella donna che indossa un bikini bianco su una spiaggia con lo skyline di una città sullo sfondo
+- Spanish: Foto de una hermosa mujer con un bikini blanco en una playa con un horizonte de la ciudad en el fondo
+- German: Foto einer schönen Frau in einem weißen Bikini an einem Strand mit einer Skyline der Stadt im Hintergrund
+- Arabic: صورة لامرأة جميلة ترتدي بيكيني أبيض على شاطئ مع أفق المدينة في الخلفية
+- Japanese: 街のスカイラインを背景にビーチで白いビキニを着た美しい女性の写真
+- Chinese: 一个美丽的女人在海滩上穿着白色比基尼的照片,背景是城市天际线
+- Korean: 도시의 스카이라인을 배경으로 해변에서 흰색 비키니를 입은 아름 다운 여성의 사진
+"""
+
+import gradio as gr
+from modules import shared, scripts, processing, devices
+
+
+ENCODERS =[
+ # 'None',
+ 'OpenGVLab/InternVL-14B-224px',
+ # 'OpenGVLab/InternViT-6B-224px',
+ # 'OpenGVLab/InternViT-6B-448px-V1-0',
+ # 'OpenGVLab/InternViT-6B-448px-V1-2',
+ # 'OpenGVLab/InternViT-6B-448px-V1-5',
+]
+GITPATH = 'git+https://github.com/mulanai/MuLan'
+
+pipe_type = None
+adapter = None
+text_encoder = None
+tokenizer = None
+text_encoder_path = None
+
+
+class Script(scripts.Script):
+ def title(self):
+ return 'MuLan'
+
+ def show(self, is_img2img):
+ if shared.cmd_opts.experimental:
+ return True if shared.backend == shared.Backend.DIFFUSERS else False
+ else:
+ return False
+
+ def ui(self, _is_img2img):
+ with gr.Row():
+ gr.HTML('  MuLan
')
+ with gr.Row():
+ selected_encoder = gr.Dropdown(label='Encoder', choices=ENCODERS, value=ENCODERS[0])
+ return [selected_encoder]
+
+ def run(self, p: processing.StableDiffusionProcessing, selected_encoder): # pylint: disable=arguments-differ
+ global pipe_type, adapter, text_encoder, tokenizer, text_encoder_path # pylint: disable=global-statement
+ if not selected_encoder or selected_encoder == 'None':
+ return
+ # create pipeline
+ if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl':
+ shared.log.error(f'MuLan: incorrect base model: {shared.sd_model.__class__.__name__}')
+ return
+
+ adapter_path = None
+ if shared.sd_model_type == 'sd':
+ adapter_path = 'mulanai/mulan-lang-adapter::sd15_aesthetic.pth'
+ if shared.sd_model_type == 'sdxl':
+ adapter_path = 'mulanai/mulan-lang-adapter::sdxl_aesthetic.pth'
+ if adapter_path is None:
+ return
+
+ # install-on-demand
+ import installer
+ installer.install(GITPATH, 'mulankit')
+ import mulankit
+
+ # backup pipeline and params
+ orig_pipeline = shared.sd_model
+ orig_prompt_attention = shared.opts.prompt_attention
+
+ # mulan only works with single image, single prompt and in fixed attention
+ p.batch_size = 1
+ p.n_iter = 1
+ shared.opts.prompt_attention = 'Fixed attention'
+ if isinstance(p.prompt, list):
+ p.prompt = p.prompt[0]
+ p.task_args['prompt'] = p.prompt
+ if isinstance(p.negative_prompt, list):
+ p.prompt = p.negative_prompt[0]
+ p.task_args['negative_prompt'] = p.negative_prompt
+
+ if pipe_type != ('sd15' if shared.sd_model_type == 'sd' else 'sdxl'):
+ pipe_type = 'sd15' if shared.sd_model_type == 'sd' else 'sdxl'
+ adapter = None
+ if text_encoder is None or tokenizer is None or text_encoder_path != selected_encoder:
+ text_encoder_path = selected_encoder
+ shared.log.debug(f'MuLan loading: encoder="{text_encoder_path}"')
+ text_encoder = None
+ tokenizer = None
+ devices.torch_gc(force=True)
+ text_encoder, tokenizer = mulankit.api.load_internvl(text_encoder_path, text_encoder, tokenizer, torch_dtype=shared.sd_model.text_encoder.dtype)
+ devices.torch_gc(force=True)
+ if adapter is None:
+ shared.log.debug(f'MuLan loading: adapter="{adapter_path}"')
+ adapter = None
+ devices.torch_gc(force=True)
+ adapter = mulankit.api.load_adapter(adapter_path, type=pipe_type)
+ devices.torch_gc(force=True)
+
+ if not getattr(shared.sd_model, 'mulan', False):
+ shared.log.info(f'MuLan apply: adapter="{adapter_path}" encoder="{text_encoder_path}"')
+ # mulankit.setup(force_sdxl_zero_empty_prompt=False, force_sdxl_zero_pool_prompt=False)
+ shared.sd_model = mulankit.transform(shared.sd_model,
+ adapter=adapter,
+ adapter_path=adapter_path,
+ text_encoder=text_encoder,
+ text_encoder_path=text_encoder_path,
+ pipe_type=pipe_type,
+ replace=False)
+ shared.sd_model.mulan = True
+ devices.torch_gc(force=True)
+
+ processing.fix_seed(p)
+ processed: processing.Processed = processing.process_images(p) # runs processing using main loop
+
+ # restore pipeline and params
+ shared.opts.data['prompt_attention'] = orig_prompt_attention
+ shared.sd_model = orig_pipeline
+ return processed