From 0680a88be3296921236bbe2e7c86d869dae0db46 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Fri, 7 Jun 2024 09:59:48 -0400
Subject: [PATCH] add resadapter

---
 CHANGELOG.md          |  3 +++
 TODO.md               |  2 --
 scripts/resadapter.py | 57 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 60 insertions(+), 2 deletions(-)
 create mode 100644 scripts/resadapter.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 882a3c111..13c1b8b2b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -33,6 +33,9 @@
   compatible with *SD15*  
 - **PCM LoRAs** allow for fast denoising using less steps with standard *SD15* and *SDXL* models  
   download from <https://huggingface.co/Kijai/converted_pcm_loras_fp16/tree/main>
+- [ByteDance ResAdapter](https://github.com/bytedance/res-adapter) resolution-free model adapter  
+  allows to use resolutions from 0.5 to 2.0 of original model resolution, compatible with *SD15* and *SDXL*
+  enable via scripts -> resadapter and select desired model
 - **Kohya HiRes Fix** allows for higher resolution generation using standard *SD15* models  
   enable via scripts -> kohya-hires-fix  
   *note*: alternative to regular hidiffusion method, but with different approach to scaling  
diff --git a/TODO.md b/TODO.md
index 8e06431de..e2159feb0 100644
--- a/TODO.md
+++ b/TODO.md
@@ -5,7 +5,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
 ## Future Candidates
 
 - stable diffusion 3.0: unreleased
-- boxdiff <https://github.com/huggingface/diffusers/pull/7947>
 - animatediff-sdxl <https://github.com/huggingface/diffusers/pull/6721>
 - async lowvram: <https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/14855>
 - fp8: <https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/14031>
@@ -14,7 +13,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
 - diffusers public callbacks  
 - include reference styles
 - lora: sc lora, dora, etc
-- resadapter: <https://github.com/bytedance/res-adapter>
 
 ## Experimental
 
diff --git a/scripts/resadapter.py b/scripts/resadapter.py
new file mode 100644
index 000000000..a70967320
--- /dev/null
+++ b/scripts/resadapter.py
@@ -0,0 +1,57 @@
+from safetensors.torch import load_file
+from huggingface_hub import hf_hub_download
+import gradio as gr
+from modules import scripts, processing, shared, sd_models, devices
+
+
+repo = 'jiaxiangc/res-adapter'
+models = {
+    'None': '',
+    'SD15 v2 general': 'resadapter_v2_sd1.5',
+    'SDXL v2 general': 'resadapter_v2_sdxl',
+    'SD15 v1 general': 'resadapter_v1_sd1.5',
+    'SD15 v1 extrapolation': 'resadapter_v1_sd1.5_extrapolation',
+    'SD15 v1 interpolation': 'resadapter_v1_sd1.5_interpolation',
+    'SDXL v1 general': 'resadapter_v1_sdxl',
+    'SDXL v1 extrapolation': 'resadapter_v1_sdxl_extrapolation',
+    'SDXL v1 interpolation': 'resadapter_v1_sdxl_interpolation',
+}
+
+class Script(scripts.Script):
+    def title(self):
+        return 'ResAdapter'
+
+    def show(self, is_img2img):
+        return not is_img2img if shared.native else False
+
+    # return signature is array of gradio components
+    def ui(self, _is_img2img):
+        with gr.Row():
+            gr.HTML('<a href="https://github.com/bytedance/res-adapter">&nbsp ResAdapter</a><br>')
+        with gr.Row():
+            model = gr.Dropdown(label="Model", choices=list(models), value="None")
+            weight = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label="Weight", value=1.0)
+        return [model, weight]
+
+    def run(self, p: processing.StableDiffusionProcessing, model, weight): # pylint: disable=arguments-differ
+        if not shared.native or model == 'None':
+            return None
+        if shared.sd_model_type == 'sd':
+            if not model.startswith('SD15'):
+                shared.log.warning(f'ResAdapter: pipeline={shared.sd_model_type} selected={model}')
+                return None
+        if shared.sd_model_type == 'sdxl':
+            if not model.startswith('SDXL'):
+                shared.log.warning(f'ResAdapter: pipeline={shared.sd_model_type} selected={model}')
+                return None
+
+        old_pipe = shared.sd_model
+        shared.sd_model.load_lora_weights(hf_hub_download(repo_id=repo, subfolder=models[model], filename="pytorch_lora_weights.safetensors"), adapter_name="res_adapter")
+        shared.sd_model.set_adapters(["res_adapter"], adapter_weights=[weight])
+        shared.sd_model.unet.load_state_dict(load_file(hf_hub_download(repo_id=repo, subfolder=models[model], filename="diffusion_pytorch_model.safetensors")), strict=False)
+        sd_models.move_model(shared.sd_model, devices.device) # move pipeline to device
+        sd_models.set_diffuser_options(shared.sd_model, vae=None, op='model')
+        shared.log.debug(f'ResAdapter: pipeline={shared.sd_model.__class__.__name__} model="{model}" weight={weight} fn={models[model]}')
+        processed = processing.process_images(p)
+        shared.sd_model = old_pipe
+        return processed