diff --git a/TODO.md b/TODO.md index 73b11c743..3674fe5ef 100644 --- a/TODO.md +++ b/TODO.md @@ -24,14 +24,17 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Code TODO -- enable ROCm for windows when available -- resize image: enable full VAE mode for resize-latent -- infotext: handle using regex instead -- processing: remove duplicate mask params -- model loader: implement model in-memory caching -- hypertile: vae breaks when using non-standard sizes -- force-reloading entire model as loading transformers only leads to massive memory usage -- add other quantization types -- lora make support quantized flux - control: support scripts via api +- enable ROCm for windows when available +- fc: autodetect distilled based on model +- fc: autodetect tensor format based on model +- hypertile: vae breaks when using non-standard sizes +- infotext: handle using regex instead +- lora: add other quantization types +- lora: force-reloading entire model as loading transformers only leads to massive memory usage +- lora: required for flux to reapply offload after lora has been applied, but fails with oom +- lora: support pre-quantized flux +- model loader: implement model in-memory caching - modernui: monkey-patch for missing tabs.select event +- processing: remove duplicate mask params +- resize image: enable full VAE mode for resize-latent diff --git a/modules/lora/lora_extract.py b/modules/lora/lora_extract.py index 5050187a2..1217b952d 100644 --- a/modules/lora/lora_extract.py +++ b/modules/lora/lora_extract.py @@ -182,7 +182,7 @@ def make_lora(fn, maxrank, auto_rank, rank_ratio, modules, overwrite): progress.remove_task(task) t3 = time.time() - # TODO: lora make support quantized flux + # TODO: lora support pre-quantized flux # if 'te' in modules and getattr(shared.sd_model, 'transformer', None) is not None: # for name, module in shared.sd_model.transformer.named_modules(): # if "norm" in name and "linear" not in name: diff --git a/modules/video_models/video_utils.py b/modules/video_models/video_utils.py index 4e26849b6..d0a80d42a 100644 --- a/modules/video_models/video_utils.py +++ b/modules/video_models/video_utils.py @@ -24,6 +24,8 @@ def get_url(url): def set_prompt(p): p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) + shared.prompt_styles.apply_styles_to_extra(p) + p.styles = [] p.task_args['prompt'] = p.prompt p.task_args['negative_prompt'] = p.negative_prompt diff --git a/scripts/animatediff.py b/scripts/animatediff.py index 34b7829c1..6fb77a45b 100644 --- a/scripts/animatediff.py +++ b/scripts/animatediff.py @@ -140,6 +140,8 @@ def set_scheduler(p, model, override: bool = False): def set_prompt(p): p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) + shared.prompt_styles.apply_styles_to_extra(p) + p.styles = [] prompts = p.prompt.split('\n') try: prompt = {} diff --git a/scripts/consistory_ext.py b/scripts/consistory_ext.py index 7a8f21e3b..c02ca1e50 100644 --- a/scripts/consistory_ext.py +++ b/scripts/consistory_ext.py @@ -118,6 +118,8 @@ class Script(scripts.Script): settings = [p.strip() for p in prompts.split('\n') if p.strip() != ''] anchors = [f'{subject} {p}' for p in settings] prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) + shared.prompt_styles.apply_styles_to_extra(p) + p.styles = [] prompts = [p.strip() for p in prompt.split('\n') if p.strip() != ''] for i, prompt in enumerate(prompts): if subject not in prompt: diff --git a/scripts/flux_prompt_enhance.py b/scripts/flux_prompt_enhance.py index 17613964a..abfbeae6d 100644 --- a/scripts/flux_prompt_enhance.py +++ b/scripts/flux_prompt_enhance.py @@ -89,6 +89,9 @@ class Script(scripts.Script): def run(self, p: processing.StableDiffusionProcessing, auto_apply, temperature, repetition_penalty, max_length): # pylint: disable=arguments-differ if auto_apply: p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) + p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) + shared.prompt_styles.apply_styles_to_extra(p) + p.styles = [] shared.log.debug(f'Prompt enhance: source="{p.prompt}"') prompts = self.enhance(p.prompt, auto_apply, temperature, repetition_penalty, max_length) p.prompt = random.choice(prompts)[0] diff --git a/scripts/mixture_of_diffusers.py b/scripts/mixture_of_diffusers.py index 463948d36..58598ec66 100644 --- a/scripts/mixture_of_diffusers.py +++ b/scripts/mixture_of_diffusers.py @@ -91,6 +91,8 @@ class Script(scripts.Script): p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) + shared.prompt_styles.apply_styles_to_extra(p) + p.styles = [] p.prompts, guidance = self.get_prompts(x_tiles, y_tiles, prompts, p.prompt, p.cfg_scale) p.all_prompts = p.prompts p.task_args['prompts'] = p.prompts diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index 8d38f5ef1..15a9d5d88 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -55,6 +55,7 @@ class Script(scripts.Script): trust_remote_code=True, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, + _attn_implementation="eager", **quant_args, ) self.llm.eval() @@ -67,6 +68,15 @@ class Script(scripts.Script): t1 = time.time() shared.log.debug(f'Prompt enhance: model="{model}" cls={self.llm.__class__.__name__} time={t1-t0:.2f} loaded') + def unload(self): + if self.llm is not None: + sd_models.move_model(self.llm, devices.cpu) + self.model = None + self.llm = None + self.tokenizer = None + devices.torch_gc() + shared.log.debug('Prompt enhance: model unloaded') + def clean(self, response): if isinstance(response, list): response = response[0] @@ -123,8 +133,8 @@ class Script(scripts.Script): if shared.opts.diffusers_offload_mode != 'none': sd_models.move_model(self.llm, devices.cpu) devices.torch_gc() - raw_response = self.tokenizer.batch_decode(outputs, skip_special_tokens=True, clean_up_tokenization_spaces=True) - shared.log.trace(f'Prompt enhance: raw="{raw_response}"') + # raw_response = self.tokenizer.batch_decode(outputs, skip_special_tokens=True, clean_up_tokenization_spaces=True) + # shared.log.trace(f'Prompt enhance: raw="{raw_response}"') outputs = outputs[:, input_len:] response = self.tokenizer.batch_decode(outputs, skip_special_tokens=True, clean_up_tokenization_spaces=True) except Exception as e: @@ -134,7 +144,6 @@ class Script(scripts.Script): shared.log.debug(f'Prompt enhance: model="{model}" time={t1-t0:.2f} inputs={input_len} outputs={outputs.shape[-1]} prompt="{response}"') return response - def apply(self, prompt, apply_prompt, llm_model, prompt_system, max_tokens, do_sample, temperature, repetition_penalty): response = self.enhance( prompt=prompt, @@ -159,6 +168,11 @@ class Script(scripts.Script): with gr.Group(): with gr.Row(): llm_model = gr.Dropdown(label='LLM model', choices=self.options.models, value=self.options.default, interactive=True, allow_custom_value=True, elem_id='prompt_enhance_model') + with gr.Row(): + load_btn = gr.Button(value='Load model', elem_id='prompt_enhance_load', variant='secondary') + load_btn.click(fn=self.load, inputs=[llm_model], outputs=[]) + unload_btn = gr.Button(value='Unload model', elem_id='prompt_enhance_unload', variant='secondary') + unload_btn.click(fn=self.unload, inputs=[], outputs=[]) with gr.Row(): prompt_system = gr.Textbox(label='System prompt', value=self.options.system_prompt, interactive=True, lines=4, elem_id='prompt_enhance_system') with gr.Row(): @@ -169,6 +183,11 @@ class Script(scripts.Script): repetition_penalty = gr.Slider(label='Repetition penalty', value=self.options.repetition_penalty, minimum=0.0, maximum=2.0, step=0.01, interactive=True) with gr.Row(): prompt_output = gr.Textbox(label='Output', value='', interactive=True, lines=4) + with gr.Row(): + clear_btn = gr.Button(value='Clear', elem_id='prompt_enhance_clear', variant='secondary') + clear_btn.click(fn=lambda: '', inputs=[], outputs=[prompt_output]) + copy_btn = gr.Button(value='Set prompt', elem_id='prompt_enhance_copy', variant='secondary') + copy_btn.click(fn=lambda x: x, inputs=[prompt_output], outputs=[self.prompt]) apply_btn.click(fn=self.apply, inputs=[self.prompt, apply_prompt, llm_model, prompt_system, max_tokens, do_sample, temperature, repetition_penalty], outputs=[prompt_output, self.prompt]) return [apply_auto, llm_model, prompt_system, max_tokens, do_sample, temperature, repetition_penalty] @@ -181,6 +200,8 @@ class Script(scripts.Script): if not apply_auto and not p.enhance_prompt: return p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) + p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) + shared.prompt_styles.apply_styles_to_extra(p) p.styles = [] p.prompt = self.enhance( prompt=p.prompt, diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index 40e726549..dbc62715d 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -225,6 +225,8 @@ class Script(scripts.Script): processing.fix_seed(p) p.prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) p.negative_prompt = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) + shared.prompt_styles.apply_styles_to_extra(p) + p.styles = [] with devices.inference_context(): output = shared.sd_model( prompt=p.prompt, diff --git a/scripts/x_adapter.py b/scripts/x_adapter.py index c67eca18b..08874aac9 100644 --- a/scripts/x_adapter.py +++ b/scripts/x_adapter.py @@ -110,6 +110,8 @@ class Script(scripts.Script): shared.opts.data['prompt_attention'] = 'fixed' prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles) negative = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles) + shared.prompt_styles.apply_styles_to_extra(p) + p.styles = [] p.task_args['prompt'] = prompt p.task_args['negative_prompt'] = negative p.task_args['prompt_sd1_5'] = prompt