From e9d2da42b26d94252c5ebadecc0639e7c723cdb2 Mon Sep 17 00:00:00 2001 From: Luca Beltrame Date: Wed, 29 May 2024 18:23:38 +0200 Subject: [PATCH 01/34] Add two new ControlNet models for SDXL This PR adds two new ControlNet models made by HF user xinsir: - New OpenPose model - New Canny model Both outperform the standard CN models and they also affect generation much less. As a plus, the OpenPose model is half the size of the one used by SD.Next. --- modules/control/units/controlnet.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index b200d1693..bc45497db 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -50,6 +50,8 @@ predefined_sdxl = { 'Depth Zoe XL': 'diffusers/controlnet-zoe-depth-sdxl-1.0', 'Depth Mid XL': 'diffusers/controlnet-depth-sdxl-1.0-mid', 'OpenPose XL': 'thibaud/controlnet-openpose-sdxl-1.0', + 'xinsir-controlnet-openpose-sdxl': 'xinsir/controlnet-openpose-sdxl-1.0', + 'xinsir-controlnet-canny': 'xinsir/controlnet-canny-sdxl-1.0' # 'StabilityAI Canny R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-canny-rank128.safetensors', # 'StabilityAI Depth R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-depth-rank128.safetensors', # 'StabilityAI Recolor R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-recolor-rank128.safetensors', From 1ec5ac1e9f0d3f635038975a5a4e9c02f73a1d55 Mon Sep 17 00:00:00 2001 From: Luca Beltrame Date: Wed, 29 May 2024 18:33:38 +0200 Subject: [PATCH 02/34] Apply suggestions from review --- modules/control/units/controlnet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index bc45497db..b7bf683fd 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -50,8 +50,8 @@ predefined_sdxl = { 'Depth Zoe XL': 'diffusers/controlnet-zoe-depth-sdxl-1.0', 'Depth Mid XL': 'diffusers/controlnet-depth-sdxl-1.0-mid', 'OpenPose XL': 'thibaud/controlnet-openpose-sdxl-1.0', - 'xinsir-controlnet-openpose-sdxl': 'xinsir/controlnet-openpose-sdxl-1.0', - 'xinsir-controlnet-canny': 'xinsir/controlnet-canny-sdxl-1.0' + 'Xinsir OpenPose XL': 'xinsir/controlnet-openpose-sdxl-1.0', + 'Xinsir Canny': 'xinsir/controlnet-canny-sdxl-1.0' # 'StabilityAI Canny R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-canny-rank128.safetensors', # 'StabilityAI Depth R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-depth-rank128.safetensors', # 'StabilityAI Recolor R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-recolor-rank128.safetensors', From 1e4800e8f590887575b22432460a63196a0334c1 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 3 Jun 2024 16:53:20 -0400 Subject: [PATCH 03/34] Jumbo update: HunyuanDiT, MuLan, PCM, T-Gate, KohyaHiResFix --- CHANGELOG.md | 40 ++++++++++++++++ TODO.md | 15 ------ html/reference.json | 18 ++++--- install-mm.py | 44 ------------------ .../Reference/Tencent-Hunyuan-HunyuanDiT.jpg | Bin 0 -> 73359 bytes modules/control/units/xs_model.py | 5 +- modules/processing_diffusers.py | 10 +++- modules/processing_helpers.py | 7 +++ modules/sd_models.py | 6 ++- modules/shared_items.py | 2 + scripts/kohya_hires_fix.py | 40 ++++++++++++++++ scripts/t_gate.py | 43 +++++++++++++++++ 12 files changed, 161 insertions(+), 69 deletions(-) delete mode 100644 install-mm.py create mode 100644 models/Reference/Tencent-Hunyuan-HunyuanDiT.jpg create mode 100644 scripts/kohya_hires_fix.py create mode 100644 scripts/t_gate.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fda471333..b462c25ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,45 @@ # Change Log for SD.Next +## TODO + +- update checkpoint: `Tencent-Hunyuan/HunyuanDiT-Diffusers` +- add controlnets + +## Update for 2024-06-03 + +*Note*: New features require `diffusers==0.29.0.dev` + +- [Tenecent HunyuanDiT](https://github.com/Tencent/HunyuanDiT) bilingual english/chinese diffusion transformer model + note: this is a very large model at ~17GB, but can be used with less VRAM using model offloading + simply select from networks -> models -> reference, model will be auto-downloaded on first use +- [MuLan](https://github.com/mulanai/MuLan) Multi-langunage prompts + write your prompts forin ~110 auto-detected languages! + compatible with SD15 and SDXL + enable in scripts -> MuLan and set encoder to `InternVL-14B-224px` encoder + *Note*: right now this is more of a proof-of-concept before smaller and/or quantized models are released + model will be auto-downloaded on first use: note its huge size of 27GB + even executing it in FP16 context will require ~16GB of VRAM for text encoder alone + examples: + - English: photo of a beautiful woman wearing a white bikini on a beach with a city skyline in the background + - Croatian: fotografija lijepe žene u bijelom bikiniju na plaži s gradskim obzorom u pozadini + - Italian: Foto di una bella donna che indossa un bikini bianco su una spiaggia con lo skyline di una città sullo sfondo + - Spanish: Foto de una hermosa mujer con un bikini blanco en una playa con un horizonte de la ciudad en el fondo + - German: Foto einer schönen Frau in einem weißen Bikini an einem Strand mit einer Skyline der Stadt im Hintergrund + - Arabic: صورة لامرأة جميلة ترتدي بيكيني أبيض على شاطئ مع أفق المدينة في الخلفية + - Japanese: 街のスカイラインを背景にビーチで白いビキニを着た美しい女性の写真 + - Chinese: 一个美丽的女人在海滩上穿着白色比基尼的照片, 背景是城市天际线 + - Korean: 도시의 스카이라인을 배경으로 해변에서 흰색 비키니를 입은 아름 다운 여성의 사진 +- [T-Gate](https://github.com/HaozheLiu-ST/T-GATE) Speed up generations by gating at which step cross-attention is no longer needed + enable via scripts -> t-gate +- **PCM LoRAs** allow for fast denoising using less steps with standard sd15 and sdxl models + download from +- **Kohya HiRes Fix** allows for higher resolution generation using standard sd15 models + enable via scripts -> kohya-hires-fix + *note*: this alternative to regular hidiffusion method, but with different approach to scaling +- additional built-in controlnet models: TODO +- lower overhead on generate calls +- cumulative fixes since the last release + ## Update for 2024-06-02 - fix textual inversion loading diff --git a/TODO.md b/TODO.md index dfac1d65b..8e06431de 100644 --- a/TODO.md +++ b/TODO.md @@ -2,10 +2,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladmandic/projects) -## Fix - -- ultralytics package install - ## Future Candidates - stable diffusion 3.0: unreleased @@ -14,25 +10,14 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - async lowvram: - fp8: - profiling: -- kohya-hires-fix: -- hunyuan-dit: - init latents: variations, img2img - diffusers public callbacks - include reference styles - lora: sc lora, dora, etc -- controlnet: additional models - resadapter: -- t-gate: ## Experimental -- [MuLan](https://github.com/mulanai/MuLan) Multi-langunage prompts - wirte your prompts in ~110 auto-detected languages! - Compatible with SD15 and SDXL - Enable in scripts -> MuLan and set encoder to `InternVL-14B-224px` encoder - (that is currently only supported encoder, but others will be added) - Note: Model will be auto-downloaded on first use: note its huge size of 27GB - Even executing it in FP16 context will require ~16GB of VRAM for text encoder alone - *Note*: Uses fixed prompt parser, so no prompt attention will be used - [SDXL Flash Mini](https://huggingface.co/sd-community/sdxl-flash-mini) SDXL type that weighs less, consumes less video memory, and the quality has not dropped much to use, simply select from *networks -> models -> reference -> SDXL Flash Mini* diff --git a/html/reference.json b/html/reference.json index 9839ea392..458612bf6 100644 --- a/html/reference.json +++ b/html/reference.json @@ -58,14 +58,14 @@ "experimental": true }, - "RunwayML SD 1.5": { + "RunwayML StableDiffusion 1.5": { "original": true, "path": "v1-5-pruned-fp16-emaonly.safetensors@https://huggingface.co/Aptronym/SDNext/resolve/main/Reference/v1-5-pruned-fp16-emaonly.safetensors?download=true", "preview": "v1-5-pruned-fp16-emaonly.jpg", "desc": "Stable Diffusion 1.5 is the base model all other 1.5 checkpoint were trained from. It's a latent text-to-image diffusion model capable of generating photo-realistic images given any text input. The Stable-Diffusion-v1-5 checkpoint was initialized with the weights of the Stable-Diffusion-v1-2 checkpoint and subsequently fine-tuned on 595k steps at resolution 512x512.", "extras": "width: 512, height: 512, sampler: DEIS, steps: 20, cfg_scale: 6.0" }, - "StabilityAI SD 2.1": { + "StabilityAI StableDiffusion 2.1": { "path": "huggingface/stabilityai/stable-diffusion-2-1-base", "preview": "stabilityai--stable-diffusion-2-1-base.jpg", "skip": true, @@ -73,7 +73,7 @@ "desc": "This stable-diffusion-2-1-base model fine-tunes stable-diffusion-2-base (512-base-ema.ckpt) with 220k extra steps taken", "extras": "width: 512, height: 512, sampler: DEIS, steps: 20, cfg_scale: 6.0" }, - "StabilityAI SD 2.1 V": { + "StabilityAI StableDiffusion 2.1 V": { "path": "huggingface/stabilityai/stable-diffusion-2-1", "preview": "stabilityai--stable-diffusion-2-1.jpg", "skip": true, @@ -81,13 +81,12 @@ "desc": "This stable-diffusion-2 model is resumed from stable-diffusion-2-base (512-base-ema.ckpt) and trained for 150k steps using a v-objective on the same dataset. Resumed for another 140k steps on 768x768 images", "extras": "width: 768, height: 768, sampler: DEIS, steps: 20, cfg_scale: 6.0" }, - "StabilityAI SD-XL 1.0 Base": { + "StabilityAI StableDiffusion XL 1.0 Base": { "path": "sd_xl_base_1.0.safetensors@https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/sd_xl_base_1.0.safetensors?download=true", "preview": "sd_xl_base_1.0.jpg", "desc": "Stable Diffusion XL (SDXL) is the latest AI image generation model that is tailored towards more photorealistic outputs with more detailed imagery and composition compared to previous SD models, including SD 2.1. It can make realistic faces, legible text within the images, and better image composition, all while using shorter and simpler prompts at a greatly increased base resolution of 1024x1024. Just like its predecessors, SDXL has the ability to generate image variations using image-to-image prompting, inpainting (reimagining of the selected parts of an image), and outpainting (creating new parts that lie outside the image borders).", "extras": "width: 1024, height: 1024, sampler: DEIS, steps: 20, cfg_scale: 6.0" }, - "StabilityAI Stable Cascade": { "path": "huggingface/stabilityai/stable-cascade", "skip": true, @@ -158,7 +157,14 @@ "preview": "PixArt-alpha--pixart_sigma_sdxlvae_T5_diffusers.jpg", "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 2.0" }, - + + "Tencent HunyuanDiT": { + "path": "XCLiu/HunyuanDiT-0523", + "desc": "Hunyuan-DiT : A Powerful Multi-Resolution Diffusion Transformer with Fine-Grained Chinese Understanding.", + "preview": "Tencent-Hunyuan-HunyuanDiT.jpg", + "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 2.0" + }, + "Kandinsky 2.1": { "path": "kandinsky-community/kandinsky-2-1", "desc": "Kandinsky 2.1 is a text-conditional diffusion model based on unCLIP and latent diffusion, composed of a transformer-based image prior model, a unet diffusion model, and a decoder. Kandinsky 2.1 inherits best practices from Dall-E 2 and Latent diffusion while introducing some new ideas. It uses the CLIP model as a text and image encoder, and diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach increases the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation.", diff --git a/install-mm.py b/install-mm.py deleted file mode 100644 index 941f01e52..000000000 --- a/install-mm.py +++ /dev/null @@ -1,44 +0,0 @@ -import os -from installer import setup_logging -setup_logging() - - -checked_ok = False - - -def check_dependencies(): - from installer import installed, pip, log - global checked_ok # pylint: disable=global-statement - debug = log.trace if os.environ.get('SD_DWPOSE_DEBUG', None) is not None else lambda *args, **kwargs: None - packages = [ - 'openmim==0.3.9', - 'mmengine==0.10.4', - 'mmcv==2.1.0', - 'mmpose==1.3.1', - 'mmdet==3.3.0', - ] - status = [installed(p, reload=False, quiet=False) for p in packages] - status.append(False) - debug(f'DWPose required={packages} status={status}') - if not all(status): - log.info(f'Installing DWPose dependencies: {[packages]}') - cmd = 'install --upgrade --no-deps --force-reinstall ' - pkgs = ' '.join(packages) - res = pip(cmd + pkgs, ignore=False, quiet=False) - debug(f'DWPose pip install: {res}') - try: - import pkg_resources - import imp # pylint: disable=deprecated-module - imp.reload(pkg_resources) - import mmcv # pylint: disable=unused-import - import mmengine # pylint: disable=unused-import - import mmpose # pylint: disable=unused-import - import mmdet # pylint: disable=unused-import - debug('DWPose import ok') - checked_ok = True - except Exception as e: - log.error(f'DWPose: {e}') - return checked_ok - - -check_dependencies() diff --git a/models/Reference/Tencent-Hunyuan-HunyuanDiT.jpg b/models/Reference/Tencent-Hunyuan-HunyuanDiT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3635f89a3c10be5cc9ac9c4cf0d7868da623af67 GIT binary patch literal 73359 zcmbSxbyOU|^XFp0A-KB*cW1GL#oaZy+u{-+kRZV&xVu|maT0uScMGt%yF-re@9e$% z>+Wh^b@y~v&+AXmOm}rv{agIE0eG(fk_Q3c;NSqJZx`U-Ccsk0$Ico6P*w)80000K z03sYI0O_p+_r~IIWdExSg8KkK_^*2Sw?=NjTMEE>W0g1l51GF4fBO7a?rLV|?7`t| z1_>@6Ha-O@6(t2JIXN{w2NN|d z8yz|M2SH{wE^ahFM>jO}|jU4H1PX043|AXM*5fG8yA)}z8p}z@0-UHy_5D?%I5s;7& z5#OW(-r50(I7qlO+*0rG)Xk7-!T3DE3Hc~=()Ha08dK-=yyk8psAz;l#3ZB)j7%Sx zS@`$`1cih}K7Nt`$;!zqXliNe=<4YkSXf$F+t}LKyL)(gdHeYKefu6779Q~{_lW={C^?)U%>u1u4MoQ0^Hln zL%;z@0IufV6{DNn#AWT0BFPa3yQ9FB!_U+Ul2&Sj=IN3W7Ne{5U17o(p(~~Ip=*Se zn)`+!@pBolsd9BeIF`K_VeqrvwsoyxPUX@wOlj83KiT11iQg6X!|#$by``;`%kgMt zZooHLgtRy(`KI{)5>+GbGqX1%IcARB)~jSipqvdq^I{FHtaO0#8Dl3}NwVxWlN>1Q z1G6Fx9X=kv7Q|Xo4$DwLd<;W;@ZG6zH9ws27jLsfC){i7e$=v=9cPY(xc0Wp?BbPj zgu*B1v4{H!$NLMXuYh$)7sCqnQ%N{NrN~6hv>GVCVmXx7HHItu`!~lc({aFviM4OV zJ1d#Bk3BVehL*;bc;TUPf95DshUev*Ubch~4prgX`)9XUE~-9TH(N=xe?v|5Peig{ znc0tR^AY)MultNeY)j;BpQyLe=-z0-BTq8SSt|~WJeuFtY7%I2JwXf_VR(q3EON8G z{RN4iC(J&5g-x7RTQ;T0SrS_gicuXI`eoqI=Xl6i?$Lx{I*}OdRH`QL`wPVV15g7c zUOUbI0Va_|6}qMz{euc+{{iwM&nx}}Jm#v)-h;0{C%7fKB9m0L!XRVFTISl|%lCFH z3I3wNjIJ!$LPB(6kwaUwGdmXbTwgN6@H)>?wi-%}$CfY)CQDkox{i#oN?B$zap9E0 zq@i{N1QzV?^?sshw6ZSHmGV19aI5xnkwpf6fNLo=Ps8k&zz3{q#7e=)C$k3_nXdt> znq}yi{UTrM{*QuCmKq^hq?h~j|7?+QXRD{?2K^H7)mZVlDbJeqhF4N+wOqwk&%u%r zJ#eiv|LQ{mpo9ULjSV)G{1GNrHK@alLeKinpogl`*P4Jh_CNAUIOfGvp7#(-S(zVuG;i|% zxus4vWq{wKMc4eEm*d6-5mGSUrCrfMRPtBPs~LW>{~K>-jJ)FII z>k7L`_oUt}S~1h~9}Ic$7V0Wq16C{QQR&4tJ}^~V`28966!GzTMc}j3KreKSvj-CnF##d6TmbJJB)sTK=rpUkFh(p&(;SUqf|E zn-0;dI^QtUBE!c-d1}J&;Br-bu+YZQ293wTYhc%b&%4=({L3hj5S4*OIY+2wKIXRi zx898Jb?J1!;>(1#OdcYx_hq?^{i?^TA_cC9BKQE=t8%^Y_Ml+5Zm8uSysk^~tl4 z*<-T!JS5oU{2%?`i5t@JT;g92twE@I$hIl>FL^KhbkQ@9qbNS4ov}3RBV1YDI&76uuBcb_0l~AW8sPJvL^%sIChEqrh5iM>(IeT*GDEvL8W%7N>;5* z-b$4@c2I(VDV~w(d;Q(voX*P0Ooi!cYTei+lK4|&D1B~_k@`*Y6!L5l#EF~@rz(jv zL*E>E*NQ~wwpKdPP$$@6b4#mhScv$!BCu{fg-?$UvdcAnDJrVU165AP(S0$EO8M+n zdF4Q;b2wU4%-wYoe4xM9e4eo+^F^N0zg+Gv)xWR_%z-+9&OIT_KecvWuD;{oua_ZD zlww8l4{)?##K2co;Co{x-##1^ymHWSKqz_F{I-@z;Xes0wFPv<#7y9FL(AbiB5T*Z zff=6>otCf@1VaiD{{FUW7;0iP1<5LGR}KxYHu3TP{h}l_CE(o$Juln_1f8=OS3g>ofvuA_4i^ES>$sCwv6h`GQc#sa5%&EVldGFmUWma0HO(oVa%z-R4d z`4E>uCdA;b8nx9hr!^vKN{0#GYH!w}DpHE*W)>PA(8^WM3CV5)-$RLxAjbGP0R!kP z?GkzIAo$4SCpT|Z_4z~OZhuDfxVYf?S8{A)MA1N75I5F9S_R*|v2U>RgHyn@f78mK^C_2Fmv-G&t9 z^nrrQzQ$d3MX*lq7dbot=v($dTTky7F=as>;%>FJdA)r`-kn6KeNo?+-?-IPfpa$k z;+$E*8Z~T8(=kZn!#yG$O?^`yL`f_xI+4r%>K^0`RydRn+U&B$9F`izj5bvm{yIG? z8y>~u8hNArO^i*1*!Cuq#o-|PWw5~!GKACCpto<-Q0l;73+_O18NY6S%+Tz>U<4VW z+Q_a3a&5k?mO|oio5Qyb$X#wStxw#NOrWurC}z@FiBbDKI2t*%?Nk;GodPL4hKJ0{ z%^|kOC!WT{Kt>zge@9}kUH-Nfw0NAC z&`X?_+t(B7w4jU73#hCOVdvSkqkbZmvy^B{kXp;VTF$K9Mn_p6-9e-n^ZW-mk6TW* zd}o>Dj_5^Qehr&Wou{SJN*_PyTQ>52!jI?T5z%S0!y#Em<(T9$3Mr(f`lz6dXON7h z5jEp3a0)vSH;~0pmmy(LCu%)2Ml_*-s4%ozTg+G^A81r`J8I@@7-n5!U~MNQx*HNL zTQy~DsJe}{;Fd*&No*=>X)LPzENlfG-qf(oPg0{$xeANr4nT1DKGRU#4eXUN3Efc7 zmiGf8>@zg?YcjBK<9sGIT&K$ z*A|Z|K2@%Xd~~wlt2abixw4?oDP3;jXe1%sm33$t{p`yvSE|+rX~^IfTl|Ms?rSxHm!mhjp?iY_H2W%3G5BY!V>wPqzE^qTv14Fx*q+e z+pPSD?wDF;&Cx7uP)K~Bj1?;ZiNNEh_W-ZNj@N(#w<2-tVa6ksNX|9B;Kj{Bv8@IM zhz0Wp3vD#Ru!EDE>RGzZaa|pUPf^}}qv{nOT}T^kh1i70%!VipDURf(e|$fh^`m_z zT4;;e>PHWCc2$*>s8)~j;2=Ro^q__jsIbicz$xc&7|T>F{*+j8$SHP*)r*SS1$cs0 zTavX0hgiAE(y`eKGp)`nYZ_y=HfG~4<2t{){W!#vJs$81!^)nMnWO2IsU5ZBx3MC4q`w!qsNr`SksH^0f-Jfz%u6jPAt*kyM+ zvMQbXh*F!~H$0tiHhoW2K8rU4HFxdw9V#VtOUSIWD0!WaYuC+H ztb`K7I37#`9XVC7YYnE%iiSML53l;F;v2bJmU|Ij9L7>LeW?(gk7hS?4)G!z{76K` zvX9-@Rgp8vm4=7)B@esEagjC0toGL#{>+>_r#Yj5peEp0q1%3;3c|ZHaJjeOip+eU z+XzFsb{u_7FMFKSdL!RI0RA8b$gcP&PKY*9sWWfZ<9tV&V4>6J$tnHxwlBIwyinXd zFnt*liTDysOxGJh${pV+nYo$8E7?_Ug7^Sh?4-&OWmj$-WKy`pbA52CGfc%oF+rA` zuI-2Mh;@HvjMO3p?}!;vcor=PJ2Rf;a*M`wTX|e)nVt+`P5W$qB5W?j01m<83V5lQ zNG3tv{+zbG;R$4|mUbu`59${L6nCbXldOPYbQH)ce)CwJ9z#`f_3ta+<3zi6JGy?kvr_Twrah*C3#_0T-K$n_(b_;#8b~ zPqL%==k)Cq>*t?gvz(|In??`lebD7&>9->S)5~~ib0Bm`HDk7i@m^Ql)eNH8p~H*d z%=`n?npg|s9q-x^uO>S5HTz;y@kEnbAnGzz*tdV=aFg|ee1XQe*%ZSlB4K_siwi9r-LL{-@Shm=!irP{}nx9zGN|7|~;e*dJg$GU|7iQtRu ziDFjZXbDZf$2Us2JzHYjrk7qF3|Hq%yy zf^cSBdctD28TWTIm!Rt`0S&PQSzwXFP&Qh{HJdycu{vkep1NM_Gi-&iX)&o%2h8n$ z9`yXIJ~A)xnkx8#O5Koo0K|d!O@LI-iq5mh9*Ccy@_u@flSX=xYm0+>0NJIRm7O%l>k^iWuH|Xk%4{24HpG{|5hdK_^C;uQ`6p5?WkuTkE z;0ZpHFP$y^P@RBP%6>Du?yj#SYC{66t7NoUsgIIF*#qwf)T@X%DxI(!a&IdR8G6+H zdTJ)9$Inr8!C@8PPNAQ37mvvKbZ6;BYPJ4lI+DOuMOn!iw9!MAEBtiw%8xme(n{;3 z{50|r{BCUn;EmlGSF13bbL)`22xAfTs+#5J_avwa>pW(`XXAdTk++7TAKX-wEIKn) z+ZW;y31<-c#sBy`o**p2F?)2cjo4uAR8gFEjgUcZ+QXZ{L{!1?sUn8XnZ7aF_gPf4 zpq@OP{1dX8EYo?x1>}3{nOm>D+|gn?2v}fu$gu!orN1Nlbb}r@G&j#((8e@%S?p>9d)+!mi z2CI=RQ@gp`Sm_yN!UqPF+Tv~j>Dk*`HE9FBeD25%EKnQ%ZdQE9Z7f$2oe=$1MMAnr z>1>}34R>aZ87-GY^QKUF61_P&)Fil)nz+qoCux^$I>RZ)@iUe+nJQ z7IMh4z$RV7-w}zvku3KKsq0$d-yNwMIrx&tBZPmG+X};?bZk=v#tZiP zE){5WzLbl)sY?4%9X#l=7ubsAMp){>G%4E2N%Y-wttck;Mdz>2nL8Vi#^~JD{SZ2dS=yWIy z97d+}jvy>QfOz&%>Fmu^QMj?8ZZHQP$;|P|ynHN3k_|L_Gz0`<+SYNOQudDSo)pEe z@_x+B4X)PFiYO~~#m0cKPYUOu z{T)Z_omXYt-#RMJu|(dh0{wOAP!~KJQ=bH`Mm{Murjcn&O7J2K0GO7Wi$N>~-Gx>{ zw?M0E`q*`${z;yicMtEmT}5%82Wyymma8F+%-$78_4dnao9{p3^RhMqj?hO_v)FYltDtXEA^1gd6UkYyaFJc8WDS?kcK7 zw<TeziNphEz$of12|dJZs1I zyn<8R7XJYli`rZ4i2H^u$lR|u+}{imH7v7v>v3wW6Nisc#DPW#(YUOfMmk-w_&<}L zWx(yRz2&tMI4auWbi&&8gX_wk{EK?(Mdqp$e?;b9`)6GY#E^LuNw&+0wf6-*C?v)m zgiNRR&372bMf9ENP5%M>Xp(mK>?wf`MJ-c}R~3$R0fqJ_?!)2GQAzO^)&f2{JSP9e z?)JKzX~OKOkdt+J2~MSRKV73oo6}F;)M?*Bo`PRC_e+J3f(H-A?2hxuJ5>lXfsn@l{V9uRr#;X!`S634Un7@8ReI#l$EYe zRK9=m?!*ULDM(6bH&1n)Hq+;Kqz0qET^8h?h6+=*GIe@3ZY4S&JL`xN-ykvsbG%#q zK)(IxucO59EsR2{Tu#ApKgXBdfR~nlDZA+Qu4v|Z$$2NHhBm=zPEHLA0Co1QP8MWr zRaEYb@tn=KO-^@Ah4cFepbC6ET(e2l9!e;1;@ds7v>qt^W=N*cW2iKi0t|Bsp!Nju5KLD@DhC25a-B{L@6$-Bv7wKNqnx+nRIiSVQKRwhbZxH- z-_)JGJ$)Pc(MyFXIMSVFfGE-p!gBMlnX1&i){~+K>(b})X&*nnSahJOvY~DpPL+FQ za0}-!*{a8pL7Pd8Rk@GffUkMq04LiB$>Ou6o2*Ipk%W`A!!ZX$;O$_5yR@oG(RnFv z;d&2eX*?`X4&^e0WDDg!xT6VzJxX51 zl^1iEjw15=!SWV5mu2a%?tiZ+9qJHtm_<6Yl`qL+C(gWSF|Q3ro=lm>B4Sy|NFBJ;@pLG{kflw=L`Jtze3Fw!kjY1@OTDw0wUDopd4yNM?kVeSbAh{# zZ&}wYR~U)A4N#0PEaZL`0>H~18D)oub2w{7(6P-h3Y6_R%NlK1M6S&QDDHuV?lU>l zTMXstgPY$aDlr4yB#&02XP2>im1XtXaq{G7Fd;SMAzvRc0$|4yq$-4KA<3gJf=1`N z71flTF6=QY0y1mOV`R9#9HsI8iKr9&v;7;QPNWVIp8Sm+APq{T_&j9w_+DfR-;EFy zF?UU?RS2Vv)Fv~fo~H!r(QU)pr;q)=q-zt&b_THNS;4P-apzBpi3b_;hg(a@zlbK( zbZ3yhzPYZ-yvdU9M8N(qJ=%m4e=?`|hw%jyr3fCL7B&Pxz!2Sf{h!TujSbTrR5Yftq=e_b_t!_vW9pwP@S*Wsdxt28 zK;HFw_M0bctYs8VPQ;o*5A{BD5AhbtR#=Jfb-^)P?E1u()A!M)wwI0!<^7EY_;bM) zvq38L`vnSjm<0A(S2_KZ(pe9^YXg_XBog%CNM%2jEGJ$}wV)S2DaheqmzKB1sC@7d zA!GCUFV>;=-4Fhpq<2X;E*ZNI+PmH4$q}yDC9v*tPwHmN50$4Ly7j=m`UH|S&e6(f zTx=DfLj~m5Ox1}Kb*$J+vbM#S=T#lIQD!N>E$Z7XI8P%Gw<+zLGB&>n1k@V{_p|=mI;(HT<3>!W z;8|x=yn46nUp82k?B}+WVX_jfnBDeGz`f;U%#ticmjN&#v?^JVELUVTRy`he?i3=Q zqYu!AjZgsXJBSj79IFP6LpHJi5nKu8#%dw$PUzIUCb_J((3yX zzETO1yP$ae1Kj)}Z5NB8w3H9wB2T#DY}$~>mThUKVgVUX8>srlSN5HV579k@<9-6k z*0acNU|JrL4zD9}5MuxQHCTm!0nW-Yy?JKMDG|>KSgcIkjwB7sB~McJ%4_7%qOsnj z8dW=q?SK0npsLkU;4%Xt?vzzxit*lUH<7IC>oM@`z=)#EVm7*(h1M+Ti$Gfbs7|CW zS|_P$r3ck@|3ijvEpsY)IWf|i7I)6QDcr?B6T(j z;yVhFXzVU61Fu5zD@7J1lC-Y^7s0`X0Tyez4Eos^>W*nb%*p%jU0H~05{SUmaI+b? z&MWNt6h=h9jnfmgsmr<@_r$>W3t}7{v*_&~kSTKwPK%@L1vt}|qx9%^C9z;VwPyOR zHCGD&H~ABTo~lLTEHa)&WL)JrM5k>bN?F(#| z!F|gM>yoR}NFiDkaq0Gk`vnnxp>D?!RHi)Y+(GrHpKS;!48Kd|{{c`r8n`7JWC@Wm z5-=7go_(x^VR?0OhflnAA|6RlEYGn;W->rIexeK=?hUnTl#66gHha^CF?vdF$nItQbM}*QcR|_u{y0|vI?DmSd2Vj;-lsV z?ty~e;nlVUyPwgd=c`sy$i9=QUN|EKQDX1uh|xG7Z9Y^lyJ2|q^m&z?#8rcb^t0o= zWuX448Cte?AnRn5m1hZ#zmpIL8C!kthnTE8AogaY47tZ|rY0PMhVNwGo;nmH$-QfI zbs>t(j2S-Sll6nTenXJtW}LxS2->Ar>)@fhtw)nMK+6w4zjv&m6ji*RU%&qcfJ9_jgjD(2Oul;VDkI|34%~s{ zBDda|a?{3s6|^B4z;r;ny$^LLT$N}t)Bn!#RSqQm1CCxd+AfEtWqM>;g;Wl%=LgiyjKLlAz7oBZ1{R239Rbe-mUM!#NRY3R^C?b3!<4!2(DTQ72jO@bW+RT$`XQ_Hx zVP*Pz&cxjNmP&B)Q{SHG^fD?>US(AsV0u{+%T<ZFBYMS%9GFA9zo|>pBTGmiZ5FqI=b60|KpPur=0$@YW&Shx+C{Mwm#hSam@JP z51o6h(AVebC)7$#8u=$5C0C81z zOZA~&GvNd;1jh6PwdGgHlK45UZU3ifc=9M?4$_xY3~Lw**_q`MI)F;uYo`iG`No@O z*-#vw5EX&~pFK676`KMDX7ghb59i6kp~GFr&;@d;>GB6II;T~hfR-|kgHA`=iuu^A z-K-qGws>5&PZmqB*%voeB&XCrn!eisp#ZhA-U9r3ew`~mjxWaLtGldjb)44A=(*5h z%c7} zEsx8Zd6wBOhldU1WNhnP$vf5)J&6vgH+}VCk-DxL;~Adc!nbwipmJT?hH|OF@;@wY z@73@72n{mzWzn3FU5V#J2YFL{@?AT*&CIvQF*xKhDsC^?^ek1RmOaT%ETg+RGfshQ zqO+Ti&$BMMpyj=3s5Nn`rnQI|@TDzBA(-MWye66vuXE>0x824y^zDNd{Jk9tAh~Wd zvXwX$HxZrYJ|H{=Yu77rXaS{D(|vlnKm(Sr%X9_L@A4$8#og)~r+4VrY4W!9@QP(2 zAn`P#ZHqr*H;=j;7U8>aG_c>mSBTL>i$29pX&;bDRgI>uiA^5&Pkf-&@w=cJWQgw- z8;Yb;q3a-x8#Bg@&@1ov_L#Vwri!%7!eBurDi2E-g)i|*zi*N2fF%l|;k^(GPe8rG zAcsFzeuYoefFA;gZ}23){RiQgmB-8JCCa$4nk30yr&pVY#VgJ~08%$ai#1`>u~)1g z_9Vs&D^nXXNBGx+K%oUbTZ2Jm7hnZ1oP#^%jmn~iFTe}1_3MnTk;wZ8JzF(8?kHJ} zo<-b1BgtR=XVn=&1n;~NEG$6-qf;_BSFZJM#CH=?CIOV04t$eP{6wNO%&66*8FDpJ z%GQ$mRM`vd&8F#R;quSM1TS03pU^tKoX|*c@PPbbzwP6`RXcy7+B_rvyWDZW=ej~0 zpPwa|`3Gi#E zM_n$x#hp30rgyE!L3sYqq)~L9d-`7Pmaa^Epd(f1qBNvK#-03X z#h&o)7~l{kG%PGcPEM7H2&1=&^V>?n;_+^0N*v$6@pVZNWbuYaMP~8$Ux+2k7C7W3 z7kGSm6d;!Mm-rcpY_4h&(59ox8Lp4J%&K8+p5{^BuIW3Z^lWNWmS><^?2FZ$^Ei<3RUC?=(?SW5m&IkGu#KUjnSk1v*% z-QH7BUCwODnItXu%=Fc|^{HC@b&QpLgo%nq8qQe+WBwXgk%50GRcHd$miq=G;|Zb^ z_1Ddn`tqrLL}+4tFXP63Gwa52JC>$8PDAk`x*ib=1Ogky{BHbJwkSt+;>ka7K4VCM z{yruKbc-$eMU~l##H=Bd1o{u~ncsXw{YRk`Yw({Pr91Iti12Vuj1l`ph1CoEEo+fE z+b8HYt#|!JQr^8CC({tSr zIZpNpj&vDQG_1A4J!>whKC?Ejt)!e_)Da{6G`qd!V7XPlYRC6nkPC=4lOo>pldImK z$$WQx`_kCARo~i-bbF(I`=+aAe5RXJCby z6-|gZv|45x=o>ekx3~6tlQxI*9kIqxI>?#j%#~b4wry!A1^O~`81wLLNu{F`w}-VaLkn8w*J zbNka-o_@LG0z1$x)+eI=nMWAmzyik|sY^LOYWFC+(-tH_Q#R@!u`l}l$&U(KN31`( zh#DHninR*ui{|`I_IR|Lbg3!IK5=$tl;KIfY!MCW@HXXKSm9Tx$tZwHHMfijAc;6K z3TlrsWFaJl?Cpqm$~f5TdQC3XA%YS(4%PsaZ>Bg~%5M-eHtCge_o9M39?KaEJAn!@ z`dE7PR@7^{vKoK=3v}ZmoMq90%0yWZ;?tuA#K?YhWU<6ipyE-@$I@ptpNd)-b3#ZQ z0u`1I@&}7HRvEel&%|Q+zlE%n2h117ka>!Hy7a%z;)5++!QxDKPhg-?h*lL2mvb>I=k*w_>o8 z$5}_C>uLgkYujw9a(r}iq^zmFzJ0cv<2vRowz-2(46M$nyeNm6>05*vRXWkXcMFJdxfyXzi0YCp zw8@*W;*FSM;X)s=-)jz)>(3#@J> z&M5jdId{Ie&FV!ZWpveraQ$6>6%H+yJBNPUb&WA~#;$~bLu0G}z3W;K+?Zl5)9f@K zGt|<{PvsZCK4a{Sxir9Z183aNCS&*7*G!5bI)c=LPLloo9jS{n)fq z&0m&FRDMdkqpf9K;UADK|Gb|xKxvgBh-`}?2d%Fzj0vy^`k3MC=P!4rBcC5`S>qM^ z0zc12R9h+8Bky!ad!~+`j^v-^IS6zpAdd7Di#15I|M`Lm2W)5dXhC%DcG?I`PIiq# zwCaADdKw%VxujGb{Fk1l9$8rl9g{N$s zIge|M(=#^Ho23xoFTVvr!DN=OHgRv2D+V?(wk!3tm?_mSN?7;tyPj}d4_JNx z$JDPhgr)EUEZo0cf$tv|n1afr+NSt3R#@}-_1PW^6=On_gAAl{vSYCC3DmhMD? zh!4@73Z&UjQeN$N>LuxLT@c;7k$>~*O&y+cIMLVZrhKOV!8Xe|5slgFvi9tEcceKa zqOQ!kQs^aX%*;ISMCtWP)YSPqXJBZ3KByrt-T75J@=LkrLC>SVu4jcqk*X*DvUk#l z7)w=QI2JcySeeiZYddySwQkCJbM|nuDC_FB7}>^?e|F%)lY!7~Q@Ecz!n>DE#NgPR zyxVF;AV}>>ZEH?^+@A_Dh=BS1wJ z{Hm;;$a(-taBrh?w8H(s%dum0HQ+v9dc3}ylFA=f#79AhSGdosV2P)104hVU$nLyY zsJ7pKQ#>s6>!~igC6Xy*cu)sOe24%T#@+o%_UAwA5p`%a z(V880i67Kji7nc8BZKfQIZ3bNI271r(2HZo>YIi?xfaRA(S11-AU6IS=%V zv4}H}jEcr)Xo0G3RTZNOmm42U8r3=Ixc{||zen!Jff8l3cUYe(0*YY`7DBap}eb@|$-n*UTTU$NOU*o3M!&Adru_TV(o#vBf?@X7e@VBbSL}jKP(Xvj6tYt4)onmT5|!GyxRNBh8> zMMs1&{6$5wX}cj)NC0b9r)7JlVl0`L-tXD(;(;0pEz8(t&?8L@OZv8bI~5g`iFy@( zWq9yzcLn{YJ!Y;UDh;k6g_EidpglF?G4HT^H};*Z%Ro8oeCX6XZ@XqIj;2GNz2*w9 z-;=7!4I{9n-{n4Y6WF&Pm=Ts7koBvPu!_-y!c4VP`k1}aiOPgc6CJzAv%1TeP zupjvEd1kC6AF=-Y==W#}LN`ybqs_KSNWl5y@9}aM1UXS3!9T5XAk|t};#%odbS@pA zttxQdXn`FiJ$lyslK4)l?`qlbjbyvA=5LAJ^T@61t7de@it@~g#ERFe*#wGK?$N}< zm{&$>&kMHd4O#bt;P z`IBbSr{-K0&`+!3caAY#+Xy-CTAPGbgZ_HQ*V!%}#f+nFU;WC}#9I%H7_viL(#4lt zX=RWIjMLq7GNs__atPUComEP!16tL!i`^7szm;wp_5WNDB*ZwOB?M_B6|%;QxUDiJUheITd)4x%+xb-O5QI})y#?^8L^5l&0>{1+eUcHW)7_D#%Fdi2 zuxAVtyb^z|zD%{4nDj#f7gLt`c`^s}aOO!#EIa4R+k~I4ATwy!51i>dUYr<;#KzfG zEcJu=+cLj)`9}2;T+@3JtSKb(9H*3%2*4CuW|K5R*5^3zpEt_Zv49iEe+g2n?sfN? z^-!XGO})A4f-{!XO4g=0biC{9*%W!@43rAva%1|#^Czgj67(7!%;=4q;ntf5Ug9D< zf(Jsgh237XnwieASeg*X$w*YOCOH@wK~7P?7wQ@!|LcWYGK$ICPr+J*;Ug+KIVN>?THLh2dCUK zEAy!h;Kj1pDE=>kzCdUw;e%Xj>rC;8?6WnS8|tZ7(zn;a3JFXItAoGLDJq#H_i*LG z*`d_lK3HwKwV*UW{ouCgasuD2pU;J?%+hQ!L`Xu#fvUIJy?d+N!vSyaKs_wgOJWwD z2jF-qNDg;)qA*3pm@4YTF>ad+wuOqPPb?JME|+!jRmIbPX%^4A8;)U?N$;Zch+CKh z2)%knaOPa-m-PIKE ztA&z=(rWERPo*RAVqZG?%BV48VW@V2)2b77#<40OenbgH^s4a(FeBt_Kh9_sK+uh( z`|7Qqkw5|Rd(({+p@VhBTg8{FfPgRtdbTU{3L7GTY@d;ywjEfE8C6Zq6&vD(CpAG;&drYj>uG#-*W6)IwxmmrZPqL!k<6-m&n}_E=N}au6VhL ziOI7FnM(6&pq~-?4JevQ3uJDGetx(8vWo_Hh@zw3LS-^5wRDuH6}dB`*?&SW;Lhg; z8%g?`Kfr5D2-$oA4f7kTJF)3F{-Ws`2MUA)8B7(kN8=Y$MPy9qL=ZU1lkN@Cuy0f2 zy@!6xsl4Czx~{RG-tG&WV1AHCQoXLSQF-LG9a6L?oIP`4KUKFu`ZAMZ>W!*7D=00V zEYDFQBXBDAgqJvfx@Zh-`v>ssgdXWVpl*aDBk=%gOF{TrFCTsqRTa zdysYXMl|Kx3w~H2FIkb_k8H}oTOLQMN|L6OH7^`yq7pQVQ?;+%9Ko%nZb=~7WHS#~ zwZ!^ka+57LfSqxNskbJ5vzbVIS2&c6b|}T-e&zf)m+xx_c4Lrpr)S7g`tg-}682|d zJtHThk3F!csVR{#MUf8cuP8WRth0OjDF9HOcw$|jVQ60Y?M?)7qV}5q{m(O-``E#$ zR6p_lP=;6fp z#Gr!0FhxKFtb8x>Pf2H3tM$A$p%be#V)U=t3mX!W17qgP1A11Yy|YHMotL)m`E|N) ztg`#QLnhFbQYD?|43$DF53Cd?%U~6&P(7ONe*nvinaQgIos#)cVHF-@lgnv-qNX7s zWH=7I9U%#SM9({_7#E}c`Q~c3`NTiV`uZ7*zhPvQ?OL%fPxFyQe))Rck^cZ7uk?&@ z!ytYsf%8Ex#gl}Ym75AdwdKb5FByz6hAe}}`pE;l3&r+{0sR|(k~L%&+MeYsEl zWk=qNZnjYwjK{rD&P6*=ZU~uoJjL~?<=c3O>$WP>>Jxp58S$XM$n`VJBGcRI5~5Eq zh?4G2mV*a%FJ+biffcNtpV@>>0b)sifn`)5iz3M|KPojgx7yJp2qMNy4AW159xwF$ zthR9u-&e9F!n@*%7v)Hh4!GK1O6Bt|8XtIrb9s+vK%fIWdLbZ6*KV6H`&N6iR;B(dT)Ug-8iQ@QOHZ`nUU6%|do(5fsISB;^Tyv*6zuNn|JP6)~& z_QQ2eq}y8*_pnwzQ-z3OI#1ycT3PR%2FTmeAKY)bpOL#*O+ax@?8QU>np4hFyC?#B1ukJ&7WQyzZ;8+byU!)lxW1Y0>bJ=p zkp<4#3o3Ij!S8!ciu2-Fb+IJ4vI!$L*^6Bg>V`ND20pM+mwrqaa(0JxcgKbFw73o8 zzdljwIruIc+T#)iM^2m@nY*ExIf9OKH`T?b)e4$6hGwWu)9iU1Mo_eTb74mSul_^@ zt#pnd9el1Z+dH9ae!M;qcLT$z6Ed2x5Wr2@_j#QuY20x zGWx3*g3p{zPBnbj3Rn`N6UF)`2=-ZiRHH}Z|78_>H={f!_LpO8sesrbUC)yO+2n=Q z&e7R2!SEP$_p2Ot*IC#eRWak)q$x)JZwTn$c| zZBk?8-}bRw;;vKG`I6;WJrOr~VMo2`n;7YL^^jK7TU#rAoOP|2CABUrY+-uR+J69) z@Ce3bP@F3^iS`d|*-@+jKP?fiRAs0tCaH@&sqPhAdIjs4-E8{&?Z9L@C;TFn7 zTi81QS82`%Cm*jhZcm}S6*YUBSN=4=)O=uu>%?pX1m% zb|4X0b6a2Oy5+{Ds_2vKo?kLZlXAvJ!lVWqfODE$?L8Szg(=5XsnF@RR(F=s+)1fV z%2=H*kk&Q~u)xUPu!HHp#0=i+xZl;a(FnBsV+Z{ElXqtV6 zjm6u;a;pAA{gip7nTZRKwa;pwXwch8)NavdgwD9QhINri*nUv>5)b!VaO162m@G9r zd%X`py3u6V64b^3C78Pbk3n1=jM|-){{Uqd#^Gh$=HB7Mw$t|l5%>Bq$S0@W>siYE z5T%CGLsI77#x{=LSD5Hd6vW8iOn0W~0vr8GKw_8(aL(mGP2`Lw?C{)<2o(uSvBfGg zi?h1eATX+uixXO{@y%1N+~P^fjaQl+v{Z^AN_RSU4-V5!#5)TMjU7sOlt| zExO+sBY2n`1y*oK`uU=CkbYG|fb)04%CWJ5=EPDJ(>z)SJ2Nd492=+*tVF+Q<8yKK@ZED!0h$cwo5d zLCE&VuDiz2q#h*j-NXwf2bwV7C?g{zes!%n-qI}XqkC$0egv?#(rs=+HrR&521Xw! zXutgxpvh zdXS)lNbjFx}Yl0UUerNu%mFmXj<{x~j8dX&`4L zai8f}S5k4ZLQNe-pNFk&HK}&Ul12{j9~ni#&)2W=H66v;%_&P9%{c{RU;qj`d7}J7&>AAUXNcs)`OHs)f(XV`{-U^V0N?5nYeP=8E#)@p3N8TNI%J;R zv7YrZPCCT6wLK(eYu*&Uvd7z?u(3j0V+l@W$EPHo-kodOb)6#DUDJd&Fs|%wjJjmT z`0I}U08aGdSNVvQ3w#I8kX^mAh^IM0(fvNPy{~BNr(Q{Cs871$G52ll!mbZO3C2g@ zYfe=qbY!YZ4$>+|BSQj9Ya_=p5wb|m2+8b4W5aOvZ?KYL&*jZSi$<}WWOgxM9b*38 z!r>Mf=4>>vZCKn7?=Jux`gQcIILcj0uaUII$~}o7kL6g-lqJ(rjjt2VEG9aAoJ|k| z1Te-2eEoRlsl|8wt1+?Gu1Z?Op;F-@4I&aa+{coY&m1Cyz3vZFJ5SZV%CkmUm<1!{ zKkE*2&$U{A4BTm&#k?}j43@9+M>@9TeDvwa?bel8MMm-`RBGvj+(H)k;FVGmA{T0Z zc=6J#XS=`APc)zLF3ZH*mDG*E zpMKQbz22+l1wBUE?&YAo(Wi6eNw#ZQ8*b1409U12L*+#>+bEh$?vZ!npJUvQtz0WL zqIdNZNQ&b1r`fj0&AkBa+A*ACA&BH3N_EP~s@p%6BI(6mTw(upth%Mxii7H%jta^-N`quE#>93<9 zneU-mYL>B^6D_2?GOfocDbG()R5b^EJb@pWfwqWTPmT{eN}l4jjHdn;9-(lWb&OM` z?wqT9HnCY(0lDXH4n2BSrH-v-sRyyVb(<^Zk2x@?2PJYj$m^Otq?WQmvN&V1OP5F- zMFX+h<~K#p89z$&Ypp*`o+!-GL2(ldEWAde*yEw}&r?>hDT>DmWo)(mN~6%OmibgG-s7zf-Aba(`|Klz)2&x#|rC& zlw+{ZU#B(I8nNmlhK+FTg1=8nn^My5wA+aiRDFurSvHWZ>CdfdnwG(P7Bt-&^H6Xt zrM4`@pOtQRwjx1q^c*lJSTNdk-X@_=S$nLxqlKR_Bu6>SjJ9>(#SFg+1eA2b-Vri_e0$RfycST#~k%!IrpIVA2d_f+wiz_*0^7AK~ zdn8B-^N>|JIXEEo_O5Butu=QteWbR3k^ZFHwEOK(P>KT- zmNAq@o^pt(o=X6{=eIc+?kfmk++P0xiE3RrO>IeFk4==_-$bSiJcEDE2V;df=Qyn2 z5O_yTwYOPq7U5BeX(jtWkzX14co|YpL(;n38jYCeDcfT^N%2*sq{7?CSk(^gl7o_e zyid~{RmeOwq}pjKeR-%QrK=F6URmw|o1xqf%sumt^p+k{S4L2b(X(x@K{TIg60{S_ zrZw`Dd~BnTeR}7mXCDp2d3~Y^vx$FyE~ydC1|1zJHhc^2u+Ql@?jVk*IOP4KX8G^q2t=BpYI)oIm#%})cj-MKL|S^ch@yY&I&{W zg+E`ozlKeCJ-6B7PGyXode>}d(_6EWRcbT6)V?@)o@r+O%(k=BVs|XKi_LYON~a|ZrV|3WM4bUU!10S+wY3E8cSk1ruRB+68qwBj_w@U>w2MrGl_0I_KFz& zzFcJc0h;&k2kQDY#)$>KlWLZha4(jzjulRNbIB*EJ@Z&PvAvN?Q_n5DHLYmuruW4< z<>Z*o;d>%ReuFvu&3bON6{ym#4)xpyRAhC?rOkcFoMpOaHD!OTNg{wn$w8bt%c*M9Q1jzqaTk9umsI)vCY+PEsn+q>ME_hXW9v3Rj^c&bCo#E~wZeD$W70oPV zNWEZ8>srdiEe}+kyi+~&b4ZNVNT5$4V5H|6Kg0(ZC!AM8nt8vBOo9ucw_v%JJ*rT4 zY+#T*4rqomo!gb8MOre8T&0^|7TY^VZ+9ZRrM81K>~}VLf=4<0K9$CJfLb_>v`Ovn zKFc9DZ)?yK$WR7*1I}?;LZssFBb{O=OIt6*^c$IDXPpy3)Tg+MgB&rbA#?mBp69)J z-~1+8mHh60*(?E+rF6t8G2xAi}$L z=g_K-gZ-mRp<7;fzI%;=*{y)g*Rxx0WNeTSbjuOXQ-O-tl2TWC2}LbQwWw?_@9gE1 zR5IK~j9H;w;N*kJ&&$B+T{IpQ(lq#yN<7mqSCHU0wg&=;slhcFZZ%oQV?RflD{=Nz zMr<&4aK!!`^P1`{=eD#!w(ulThRzXj&-m7Jr73FN*qub)ti7P$L}h5<4hcESbMn=j zM`Mk#Y|l8`kH(@DjTO(7IboR4WwiB!KaO@`2u{!)}4P=y#{BLwF^QHmW%r^?t{xn3rSK)}Vh$@y3} zLG-C+f+!|2#NtI5`G-Pr%@eOaVv44-F!3@Q1WCm8?^dF1iwS+x|s@~p(x zb$g9oaU|~FDfv{kKpltSQ|dk+nJ)DK1T7Rc;5W{4IRoVfxyh=hI(AKIaar8MEE|Gr zZ(eJEHKL3mk;GjPPpHI*B zwXs17z-D7J5R3BUmd9KXoP8@*E2}d;TRWrK?WRYx3mQtETNw7r2xZ4(pXbhLEhmcV z-*QV7pyMo=9OJs4RMtq@nc9Ytr)sjrCZS~w!^954;N)kh_a4=)u#dZnq}te*6!$#C z$9@`)D^Yz6Otz7L9@+pARQ~{b82k?v=`&i#Bu^aCt483fDFA{ygI5{bb2-vdS|2n= zA+xySK2bccrFsX8ygz@X-G8A)y25TFFhHQE`z?W=$GvR1X0$n5oi}E4c5+1^XU
P=|<|}>B^QXAjE#(Fzb%kq2Vet ztV?z^4g64&jc@jOW;v1~dt;rXAIh)8;d`s(g6|Pq&#)Mxoe3xpb|b&H3eT~J zairN@+atRdnkh1oNO$Acj)&{kr)e&<>ExGGwlOM(Wd$~dCyu?%CGDW@Du;QdGB*Hu2=7 zsZlkls>~GP)RoUzy^~6@w|OqEk|Yptrm5V>6#XqaxZlEiKJpTwDGK=(vzNQStYcrv$gBwF5zn|VSV{0Bl4@1 z;3ZKNv|U!ld+Uph2SuMwjg?uXR4OGo=kAU=bO#;i8g!b5xYt*3-V1b@c1A5%=3n=V z@H6dMw+VF`t!{>XudQnOPQJ70kjnyG$>v_+*%DKXbjJtZy=`lHb-c>+t%J`da;Vcq z4?d;2%{a~*Gda|4_BdUB`7U+)YujdM8OT}WA=t=rGqpkLS5v$i;$)t!5{I!27kcKZfn1wf+2xv8XO{ zaO1WQe@f%M2fPH7maOD-FBZ!Vxu`*VYPYdpW>kT#%P=5;z#tMZKT5X_fn@?G_D-Ua zyB2v?Ws#3K;aCBjSA1PItd_=+tk%Yj-m7ggMJ|%9rRpwxyoi2Ixj9j|=iiJ~O(Vl` z-0ENH7guvACty{R83(V;(-mB?w#6#*Z$j3e;vHT)Hn+RJv`cIiF__6Y^Z+L$pIT>* z^gSla!}3h&de`nha(0Yy)V6vLan5R?4tHw})|^(SDXCjoYRu8TiD1_QIbjgolE z=jLDz+;Nf^XB_cda$HGlTgbSU65UL470=~eN{x*Dz4t9yTI!I5Q6jGH6=mf0sz&zn z%vU#;3{itAX_(-TsjK#4_OGKmO(XsijV}Derd8Z=BylR_Cn;iaC-dbs}IaisK zFQ=_Ul`C4pPOIuug7VT#Dbvd1zWfi_JDVLQ z94u3pK3uGSh~t33@B#F!yoxg5kWFt+YGo?qi9XE30^!m07skz##PI zxhqc+EKGjSu8@Qb2=2=B-+_v12SieevLLyF^2bW@rhAyeZSx*o`Q4B(H+1HiulSbk z_Rs9U+YKy34$Ca&MPd#>!8q;wsj6<_IeWH8brexgr!*FtyWGOdjHHTAG3)*{<{Dqb z3A9~3Y~r^>Xw;D^#Ei|j;{D@VNbIb`MVy2i2-@=JEA7wgXz?J(X!e-Jj8HuJGk#6vQ>`Y zWug}|?0L_IJSXEH3ti0c>UwfQ;}34^(?u^GNEpE$qo2aNdD7*j*%D=suLJQkrBUod zsPANRx`wUdSRNOXL9>xZ`ow~EaVKn!g-7RFpJCN5rVZpciB8{`V4uRdr8j*M4o|Vq z-uREjbJ|F$rJD#b%#d5KC^~S&{=I9x+SYn=OLBoAMg8MGHjl6PQ&)WrHEQP*;WoRL z=^s+m?xeT#3w@u>CwqFgx3_xgbc+op38lHW+b5X&kPv=FJrzf-Xp$5YzO4DSHJekt z3zXhRbASN#tJ+1JF|u7;Mr10$`A_nI4@W)NdkXH9)D@z1k3y-dXui z%nx(<^%dP($qeVpyp>4Vv`GN}09bR8$EeRwO6K~@Qs!WqF#I1APaG}wMJ^@zWY_^^ z?l%r|+PbS2mfq$vvqJdNEp!z~{=(N6?mXCz?tt|LhZq91F11#) z*e;`th)&o9ZM(Pe9>SI9Eq4o?{Mw!Vkqm}wjndp9Q3wa+=a5g+o;v2a`|GiJs7-L? zlXEjFsm|X(J*%NoPR>Y+l%{uXkPj;5Cff**^-+!i?eph9rA_-6Zs%`kZx-fA@4T5L*f!jv z>@p1WW$Z?KanBg(S^6!^mzw3?={(5efn!nz(VXO*cFE03dY0CP*~m2Vne9|a!cIES z(UZ;y`W#Rz6>qHJ)NJLnxsS|-Roduyt07r)-|?m=u8i?77Wi}f7WeG1*nZ+qni;n< zvyS7?5%jNN)b#Y!?4-4ucu5zAM#C?-KDD%KTXUXxboGhzjF8;vlE%`*BypD971td; zwbEJ((woUxXu#Y%8?Xo4HWD37~4*ed%I?^sj9Egd|W z#K1P<6-WU7HJgJ*(W@48DH@5TxM@LWJ47Zv0QJw-sM}wq#?g=%9z%5>N}WqqH2%)G zX##Xyt`14+KhHG?yin_IcOA%FWqJXQbNKe4W?rRj{-3zE#UXRO@H6xyHLnGwh1R3y zYPUw(+h$dSW%n@co{BxlqT;a}2J+aSvEJQ4-MqP*uludr^QlBbb^@xg9D%?mo^W{l zjR9jq+r~Qmw5lX%@6~hrseg8J>PH#8JBZ?q zq>c!O$}l?v)ceuwkuR5`FHfLZ%;jw^?FHiT<;MHtkoLxU6Zuv`BsLnn4FgRhDMA>P zIbsG!Pkd1*$t%T^WORCegzcUWpT$}Me8hq%S5IU{wew5&_&Ceih7W5$WS`)N8eEvkU7hS& zZI-A4+ugRHpo56`8T#z5S3|m0x;k=8G_eu9G>jZ%e{>Pk70)O_I@q-})byKuI%Gu5 z5=Vjm04-f}R4i>_mK}_z<>M+sgZ}`I*9AyL%I1nm>~xUYEVk~C$ILo~B>qN|PIZY) zW+r4nFj`Meov9|RWNQ|NYn}y==Gwb{qa1D*=>#d@)(xo=@O z#hgZ$;i&Yxc;jhaMgx|>88z3uN^!?Z@AxR*H`wm+?PJ|neR{4;$SNgMZh#7nt}1Re)^+Bjt-V>qO)j32aGrN@@& zeANe$89&OiG`kF8rCZ^?Ex0SUY5eL|33X(uj{6VP?h@&|g|eDR#tSn<*gn{&t*gs& z$fjUG&e9l5TE z#Ub-{MtoZ2w+@SA0;@1?B`0vlLyrFdg--q+hiHm6RM-XwnVxai*!zrEO66M%nq5vy z#Fk&$VBB3!@m#3Dk(DM)dSHS8`eL`VyKQm{oxW5zmgqipMOpq)oP4!9^74vUBzBaSj+r_vqf0F|z@%(Gq5<5xdWmvT6WoZ@6AjxRkM^UxZsn4!! zPiYRtB|hhb%{7(0zFf_4u7hsY(;-v%g57J?ykFt?rPM86-p<@dbgT;9&k2aK$OBuR1kC#I$7M-Z-@kCGDnPOxOR6HAlWZqCpRLxgX*`K7t3{{RbiD`U24^oyJfv9y*}!1epJf}R$+ zO?`A&?Ch;>O}E+r7U!yA3BXGNqQHLk-#Z%<_D&%ud zd1!h5o22+Tp^5b^Zt6Kog7GDx0Z(zeoc(K|);x0@R>&X0`XQW*i5e`ZKTHqjT-6m` z&3w^OaeYeA_-ny>mA{z}i6o6%_$3P)E_M^+JOw%!}kzj!}(j0+6o z_(uRAde7O({7dpQrAkgqlU6eO!@AdsE-p1WHH{ucymHK`cA->u01=!HpM`07B|Jgm z6R4jltZd<75km3?Nqc|^>^V91rm0Fl3P$v>v#l*%yxGf3px<~yQHJwQ@fEChSUY61 zXJ%eG2WpXn@7B7{9QY4bd)rC$i6&@g%3N-pg2uz0$0vYCr*ZFFsL5_r@RcDhSnKF> zw|+M9T58spsjIYdJYWbTAU=sCy~ ztd~O^O$xYXxz(<8QQi2HUS&6W@mAhT3pqfIhH^ITAY<{Xo*zW8hBtDG%L1ydM@$ZV zx_-6EI^43+9FmpZ=GtkxuA*Q~R{D7@B0H5{UHCnS9ox1|OKE*1^KSF88t^gGJ;lsm9V+@tcwKDFCYT)HG{p=s`~E!k~YtZfl5wFh!CPvt_+ zlU*e2vF$CyeBd7AAJVeEnyo~18g8K*Tlu#WxMLangcJ3u*LF8I&9hI@hmO80=uU&fh?2wD2ynW8Wv)?WH`SXWRmxryr4`R3g=^ zH5nT=Gns|Kkx~&9Y`p@=+#C;lbAwKc!BTyd66;hXsT-n#JRA=BJ%1{nt1X73Ba??v zVXDJx6pI8Yo$b3A9m(}RwdhtJ71Aw!@0jyBKRmK$c6#xFgIY?R<7*95XP;`+~Eeq3&ifc-E%IPdkZQHw*|Te}ez$ODe} zKK_*A?(9R0TakEk!Qb0ASiHQB(jYB;Gzdd5+b z+}<;b*z`?W?$#?YWXT=emqYh(IU7j+_Im(orG}pzX&u8#*FniLPDTryi~*i;SxRXd zsJ3YMdOt4?7#ERuvlG;x=s2S6&u?RQaba&9_XrosjhkxnMn-!Mzm-g7b~hajt=n*9 zXz-_R2eGbMW$~P|Fpk?ohj0fCBe@6aI@YLzs}sUWUo(-6fHPcdR(>#&VKlmyj|*+a z7SYe=$f_jl9H;le_XMB9xuw&5M)4y{uUp3535LN(*Y6WdOzBoYNCWZQ2_V-s{{RU7 zts=3T#M+9fIc0_@qsaUgfYj91bSo`p$*u2W+^3@|fMaj(1KP4jgfVn9gKLn2(p$QE`*(Nv+wu9j43vwp~g>e(93_ zQ-k@{s`eoX*617ZrP+OfDt2zE!zwG(n@mAoU|7KZQQ)OAxz6&={mchDI1FeF&>` z`x&aKJ2O%l)W~ynOEMb@CR$L$9JZ{H5Iv>#0 z3tPmMcJ9hJ+D>p!Zv1~EQDW|DjFihNl|jxkfzSQ)wU?vb%KB7j&M+O62Kh)EvM}Ag zhKmm5m++gNLfI|sh0L-Nk>&dP1|53$s}ezLco~~=NC^c_75V5#X6Z%6V@7`s%i;Lo zR(Qi*0$C$-DlRe9u=nj$EuoStz-}*PxLtu`^9ff$*8?Zkky;lcwU2XRIHj2!U_mYq z9e6+CQ_T!NXptN&lB+30C+?ZM{SOr*D$cP+9#1yn+iQ%+A`$^P^#1?~vmgr$fvzQl z0tt%X7E_bkukxlV6lRD@s4c`-AG;fjTrfQ6jt{45ZPLRWaY$`pCaT9!;W#D4}ONIlbqbpjK`_W zFaQKA59`G%#dZi&ZEU=HN7M7BB55yH=KdnLa}Ww34yT*|GI{QE`PJ8%JZ?L_* za|J^hN0w8Lhp9OIYGnP^F^y>~+J(fe83CQo1CF)m7n+BLZS=z&Ru__L3V}h8kh`(> zk3Wtv?Odu-va&g3yv}YP4eEDRo^`d0O%#B9$N|Pb9E!KLYsl6|c6<$>p2DJ2W_L5L z^y|Ag(hImAE&~N*1NHA$Vv%g`;z@#eSS5=>Bt1Cb^{7#E7Uuqe;capx5}QVj7Xbe8 zlxN=uo|TtrtKJC*mgyHQf`|}f104lg!#C6}k>5pkX{V7K&9+`QC{lS9$F8HP<>tAX zc_Ao5G;9FQdi13%6MaYR8rbg+s}PS7!z_%$kf?Fj>0VmbH%KFKeIkX=$V(mE)T|5c zORYDh&up~Urnp^8HIIo-4z_4Tb)DvL(W zh^mch&1)2tKnK_qMC2~ezFrs9?m0E&y4S>S5Si~GhVku|8E~d0P)NzZ&MSD~M5;zQ z9@+L72}W4Jc*Y|<;eQO*&nLxN{oT!~&#hmTc=wg_jtR?k$I`Fr2)FYG)o zsmrK1do_y=ZSao3kT}k8d9ND#RFY^uB57uMOV|&YxXJU=w`yvIn$(3+O6&DHofA}K zyGtilFzzaiEBwkm3GZHOrR%p4$P#(pIKcU2Sa-6X*clZyF#24l_fUa#@?91CA?)@ZX7iMLpye z`hXW{IaQBzu}9^cdJmYK4wW@2$yuebyeAf|W3_7;)HLB^3ZR|EMNKZFa*O1fAZ@}E zE3BcIdj!T!Q<6nmC0kiMsZhrf`FR_H#xj2@pGej;>0^7QklRHcd6`!^IOJy|@v4O4 z(2~BlFG&rKr)W}pBOr&7mBTOlr>H+eT2}WEUO=;3PLjmg0pnmn{A)|5Xrpv-?coU) zYkTc7cv)c%qT2ot!GRbcy5WzubBgNqE5y@u>F*>i%Mz2cV=Q|C^*9s`6glLh6?S>l z`Vyy!;2)! zN;#$6bHh#+a#p{Q&Ix=MBl&XbcUzBZi@SyY0MLl8j8A8Gs!gVSu1Rhph)E)pQWO)6 zXB~aL#V%=gCDEFXHMI0OON}SOmm2Pu_H%DEF0zS_+L^j2E0S9yoYzx7h4jmp-z+Lv zcM7V1K}R&q-8XdXj0Ms3_?WCd9n)NPFET!VHmw~;MbI>>R-Su#TL6_rC7G}@z#q=C zocR^yD+eUIe_wIWc%wy4gw0rAA1-6x?Vl2KvmPa_x8Aa*%*Fk%CDiGwb z+5ld0fI4D_ic6`Sq`Mwjrg%2@SZM7m^!V}iRF@1@GOWZ0AfIe{@tX8&eNsrT`eF9 z5#FWLsb)x$F&pGaJ75k7KBkrX9MYxAxs?9^uaRTK9u3lU!)JG@HNl3_WD5iYFPK=4 zN$cCSKJNM?Gc?ogW>Bkd8)pD?$>=LN)~64SNgPv+DM>oAxz9eCZ>wEPX<=-ZR(8>a zc?m8_$inB?0%|#Kt{&LJtRjV&vjiB;c^HgsHva&2j(lA3(u##BThG7z17a@@>GsH^ zJib~lIc$DCE1!yMk3CjB&z`7B`M$M{JY<}$V~dtO5ugM@Y(iY8ZQW``jpdhlMUSV- zodtR@sLAC%ezVnWblXNqZ6cOQBiiOrNy$9^9@UA@LEgLFD7Sa1(POI9d_@kIsClcW zKrN(HI8+{D9CAVHipSIjR>oa~Z?;^LdXPx|m93E}+hf;kwaZOj-DI zRgfSjJ@by=g-eSmO&+;rZEt;L_K1lKAGge>{jc$^Vl8IY+Bj~azF5{p+=VmqkEh}K zR5WRGq-zjVvSFAUh2xK>_*Xlqc&y!^lEN=8LI^5GcMy809X_7atUhFP5%0)(Vr`%E z(f&}sQ(VXPcDJeOF1PYa4VxkzvJ--Kf_AU3Jx{GII)(4C(dv5To|p4^sWYPmD-X;= z9+^D_RkG4FxNq9pOH(TCE*j!Z{{VNV7z4daoDoVhW>&X*bFFEY2H!{s8r03dZs^9` zklg&B{I%K35>IN-rr#!5iYkG|2AGskSeZJMG*cIv=7gzsTG<`l#6In@`Kd*C2bCjNY65^M$Vc)bx=cjMykVL~%xr%5%Yq3VF-?&X;+H#IassPzMml|J(20T0RaM*( zoR%XUssU+&j~d9Wj&Mlh6bQE##Q>ZFcy4N;M?&I9U6=%UkDtSqrXt4DqZnx!I1I!L z)G0FV^AG|@2M!4SbO9-fM)`K(8>t78Q-=-1YzjwQ_xz|9rAD`4GsfUL5yAfe>(s<; zZLRYW$nTN)P#T=pz2OTTen{rHGDSGx%MegldNxPoYob|DsXknQ9;@Y^zm-yxTN5Zo zTcgHx--p_rl(!OFD7mzZZdcqlk>BO+N40wVkm5`QJ4nc3oyC<@bP$6VLE&EY));_6AJ(&F=*=Vo@opL_V6=HaxTZT{?*@};pHsm=R z`{VPaeMe)BOK~KAVv-Q_;AXmjVKS_@5D<5V6U1=G2ft&}z7M@xlIlZ?=veT7hwK~; zOH!G9r8}aFcnolU`St`>9mbb)X>%O5_?ij7dXeth%!Hm9bJKz;OHs#7&p_4eGklXT z%@o8kIOxjUkLO*d_F2EttX0M3TjC_L@bNQ5<2w0jsd$fj$_JlR!I*Si)&-J zf+IQOj!$aURP!53+ZHFf;+2f68#(SLvv}fmM2~ASsUdl8xcb)M*PhZwid#fz*ll3D zM+dkns&YQGxu*6+>D-}nq+U;9eGGQe$s?+wD#miha6AsR)c8{V4O0H@7%Y6Ll>;P= zaNw}q00i}3qN|=sYB)ykMr7#U556L5Yr0Fz%{mnzX71_GF1@cqJ=?``!d zE!~q2q26)NToT;=Rg~%&j`e!fU4+2Xh6ODUT)up_*n`5L}gNV-=#~hxu==S<( zihGBSX^_g{*bTg#W7FD)QL^Js&D_qm@Mebz-YuqQ{`8kpu^&QPis%st9F{*?hpy&| z3$ab?;Wpo9Xv4%88(5a(>UpcheMK4%l+U)(tYdMu!r4F^5eokRg)X_P!=UKb8id1e ziGD=R3%6wzGL&MuqzWk#-s)Bs8z8y8hSo`e5r%N2@ziF#Uf$N@#(ItA(Fn4a49bnj zQ|K7-2iNkhb%=F8gsyed!}~~c!b!|!_Q*XBIPN&EsL8devppE%X(@9?3M|I{&U1EdB6mh!5QBmT zrF61F6c$i2c+d5!O&Wc1waQ~be;gOW)+yC5(mmxs{u9U3@S^I%-%Wzz_{Le)OOggz z$RDXRHmqB8rmeewaV?gir|37L>i%ymN<6LjCzc@O`+U5b=ODe&uk53_yuVo*S5V5p zgBkZ7zP$kNQ_mFa*zpvpT~ow1y+7cczl(1!lS{QaZNoVajaa+3`-O@Cu$>Bf)9=w`~(S+=uLzfigT}G8C=#LTq0EE+7xQJQD ze<1T1C}5yJKkaqJOX3|rPF-3XJu=-b?W29KASIlhI`=#u*0FSH#n|J;;N^v~n_SPL z>FRg(`i0CIW6N=>EU`DsEVD=-dkzWbIp?)!SlDS3>2M3DV!0qd94J6IQR|-m`K}yA zc+-7OXvR%m;$dfJxeB zWBiBs)m>KYEz5J3Ju9M&6!wYBD@Uoj;olKSeHF)_Y>+fy0MTdXW7y>J#c{Tp{l%#A z!p2ncqi;`q_pXWI=N0ZFE_2l_E`*Qw@m#fry%&2g34r+giv;#q}0n|qjt{QM!cE}iz|CYDyuMHM=~}@{w}?`3|CR4 zX!g1^u}rwSyABGz2t@w?baf}()-G{3j3jE^!tw}bUR6%h&t@Z>cjOPwpA5*%!QE9= zK_{u_ryqrLb2FmIhASANkV`8WW($mw)1Jfs0M}2NMjl(jq=j*wMo1uo&*M?6+`SQI zT&&Q*xObN$V)Ym!{A)esm+euZI~C(&w2pXTkA8E~lS54M_wZ(|b*|mOCb!eAg~N!8 zsNCte;HdsCJLbI%>v0Rqtt$G(3np zrB8DZ1Il#=zXjKK=uH=P@aq;gQAnUf=u|1~k6*_X%$jdgp{H#MdZol|so231pu_ow zP27-h2e)qZZqDjCTu%ro5ekY7c_e!0>svw^SjtknxuyiaX^RsnO|7{YjGmar2iBPL zDzIZDxg!87433=8D+S* zTr(3GS%^D*FaV@?T&=^+atvUQ$qq4*&uRd-5mLJ&1=N3hx&2RC6hBb&$UPZ={#c+y z=_8WAy9XR#A;|nms?$v@dqUeO1mKpN82u<2Jn<6h-d+gjl^7t^IKWAnnXOe==LL%q zNC(LzNt7@~6^;&e_otSHkIKkT`zt^duA_?P&Rbdbn59D-1(1S49-@ntVic*9Iq0d+ z@}l+&C!;)P#C{)}M!qqT=G(qZQf>Rg)9O3ds%m;->v~j{cW_U+vQdsOjo6Ryu83B9 zNaU%8Tfp#{A;BQzel?@0*j(usvRU54$Sp2sUrIx_-TaB{st;DJ}A zRV08-ONuHpXoWQ?7ZJ>WulG*_@c!xgR1-(eFn*@9a5jwYO>tz`7cV3?22PAO514y< z;;QeKHQT`$<2|Z*OHy49i%xk?mWo7oD4aBB{bFOd?~Hn6dWz1r)CZPKePvi2!M5!X zED$`n2X}|y?gV#t*TJ0t!QEld;7)J`cXxLP?(UX1=iPJP_x`l(uI{Ru`le*p+H2+5 z{0)4zFl-m-_y@=q(fnQ&G&)n&Ucm`=1Z_It`RDGGT$S6ijb>yhFT}k(wI*FI=T;*_ zi;vZ2q7NG<{gy!``SA}RrVLK#g?CFO6TTH5Lp6o`cZ!VH6W`6VNt-h;s6r5_R2@eH zN?PtqO%vUIhVRLk!h0l30*4+I@5)Snybws=N`>8Rt&<1jOH7U*?qLzf-yowZu8tR+ z)f#>Be166*Eg7Dbg#==qE}XNwRK;aRx-KqRQS{%TiZE=yg;ub)qg~w(?<>-HL)Pis*$y{g36CA05f^DU#!C8YY3CT%_V>Ihxn9*Z z7w_4gn9+%^s)%PHp`)J&3}?v`PZe3Y^~siCvI6<;c0sZ7vgZ)YRo$loa<{syT!oZE zb7$R;N-LRCFJ7%gK2J?MZ3&ZZBoug^LN@}ik^J=(pQLhyD%{NJw>Ru>EK82)XW~Ou@eQyR;^6ozXan*`B z*e*LY?^OrF)Z0I&p^E`g%R!0jeQ4#%M(%&uve1Zm7$bu$^*XPqtARSimv8e_SDm{kp58)Wx#jc-+AL?>DebaR0H zV;>yV%&xaMPkA!}eLBbV#FC8T^|j`LmAoW*Z>KXte~G78_L5$y20$v>>AfXDfzwpA z)8^~gjkw&JM*6YE4KEM=ehiUR(Pipt_SueVh!jo^POF>L91Jp*u#bn?;qlyEwPkSW zqsP}D*ZRy_;F#)~e)Hh#R`k1NI0H=_*tS_3bn2Y^zHD z0~ljaR}vyeUa*62C`Q>CgW1&9c|Z3@6QbNve)p23O$>9Domxg<)4DfkMV@A|A}+lt ziK5@eEC8>AU-gh3TC0*ct3x1!iZhVIm}*^f-^=jVzKUElhMdrEVHbeH=J&d!>gdK4 z{gMSTOu_u#{K*(C8kPJr4s`7#`)u{e$3wx(6jS^7LQ*dmmnxnLIfH`-yAVUw`t>w% zI}kKkyL1m1Kf<`%2K^I-Rjqu;mI(7!Cw1R``DiD?By0@c)w5g0+Yi}ut6yGj(7H-s zF@mWYQ>#+Zjqk+b zH#G+!tZuGHB>m5KhY3&UI_1u_!Q-^emsH+U;j!R=&e>6J(LaM{@d51Jz- zw%RbNH1FrNgL^{rK8wyOy4k=w5C`(%dmHiTBOtiu}I>JNQU z{gUx-z%97CeW^56^zrqaL)!0suYpiEoHW_p!y4hGrpH;Jg-mtN^wySDcy9jOzOKJY z^FIV@za}+3`R(v!=uV_gPO`0xjT*)ob;Y=TeX2Vl)!ytbK9WE>(|Z{*^gA3quBuHkl* zi0H$hjl@eEKdCcmRglY#N=12=^jY@lfL9+eL@11RQq@1myzmHtA6VzDjrj&g9*eU> zgi#bIT;om|Lokqu%H4b>$KLdnJ(qa1=n6lvi*_7KoD5s45`@lKk+J*ut}KPvet6%5 zmj3}gsr9_@AE25w_f$L4x_LTiR(mqCfTF<+K;I_Tu%)PM>_*%T->2;zl*MNet6zM`5|!}vl{HU;y6d2A=n@_gyxP`iyiwpT%1F{DRK zD-}*|BVK<|rT~3D8oT!UJr2B3moU;Y-M=3Z8diNJ%AY+-wjR~P32d%Pb zj5oRad`rjxRwKrL4B*)Vn~M;&R-( zLNz`Kl@O<)PEGQ3Efilx+xwC+sWDykRR$15I#Y^^E0P0WLRICL!atCZ6>5;a)5>7( zeA2m%puJ3@-26t@I?}n1<2xS4v#bD5=!)e|(Ma5fG=kBd=txt9TwO{p{X1o$2^@cjHKBntM)~s%2EQtqg`NJ}HGa1m`pX0dv5}r@8 zZS7E$x=zm7&-&X<>;^+p0p1%%{2PuER;zAFA9`>K@{tqk04j&q^SzYNS%1_!RGwLd zJr1CU-8AsvheE*hNepSwvF}38iTQ{gdkfy>5GItp{jOr0Euv3O*JvKA6pUT({IHrC zbabwR7253=J38u6szU-yLVuWfuGymZEKMiTh!zVr?>N(;W8txHy>u0uaHK|8dAXyZ z)}!%3EE~;mkOxRU;VI7MX?WdqVK>L>zQ;`8j5+6(oz-XZdsLqq0a~L8LCuUq)CW3K zxk?S;NnCA&M((!-hR2{!2La)}eD7e>{m*y~`4oNZEJM?Khwy7%95Ordh%8TwzK=ry z{zTL_Cv`~$rlW{Kb`8sHn-BCw*G$I-y|r(@{0<`?&tI-*+ZLAW%pdgwA55s z%VhdgL(DIr%CgM`3Jk{=&@v?AV?~B$i|7WL+O}U&kj11IWTImqY@OT_zdLouetuCU zk`01-?Sto^hj5pXn+z!-^~!nN6isbR!5;mX(Bj}^6;*2#?yPA;y#4d4d!Oa9YZVLyDROk2Y*V!(jfkJ8avI^KT(D7*>xwlZfjqR(AXs*`Ta2>O0SgBjK(xU4s!mYRevk8#zu zrHlMBrX}l{x>0-?=>X?i-}aZUOIKw!G^0hlGACz71BITw1kz%LB9??cCTbY9h#gRT zey@um6%tiaz`Xz)UGWojv2_;xCw@}_8hO{xgN!~^?*~Pp8D6lZ_>FIA>^QGi6J&HR zs2x*_El0jG-hrtP)>QH>=G{~!YvL{Qkvase4P<+kiM$=dex$0aTI{pvNZP z+%!7IsK_>}o_`8PWt(2as$G)dY(S$9KN*4)u_N&Q9t-4aZC6J(vhNIST*rn=o*>H| zMr5}%e3$}D4Q?#dBEs$#X$P|6F@>*! z+uSVFjYAC$9X}eK>ODbc{LwQ#q@*D#pTEecMUg3GEQz>7n+nmKUK%(u$1>TQ(IdAb zZSS#CCi@ees((2Xr&a4LQQjp9$MH7IE?n-P*^oio;Bz9^AnE1H@;IU&ztr!VfjHU)HM&Iea~7Z6>d z_jXBQOr8RE5TBgyHmajw&(>ISt>Vvr+>=6$yxf)B>xvqOWo2)MwQOk1(XpuX5uwnt zOFb3*?eO^_*ES)!CEq%^!suY$qxvy2i022vNLDi(r4nIL&m1Jj7s5T*9e``X{{hgq zgH}hG5l$~3dg$>s7o0O$yBVaY1wt&(^uo2cJ0=sidcA+0!PHg)58sXbr8;?+(zSd| zuR|hBayg53z9v%t{InIhJ^!xkW)a`pOP&7&+LFJrkJgE;0lIKjjG`OWMK*qxNS$;r8@>Rc*U#RGp zJGvPR8Qnncxn9wCNjeWS6d15`hlQc%-j5TeSCCFlCeN2D_6*)tC|`Yx`5ompyJKHl zd1YQqvgvS|D-x}l=0(vY^b6vSVUU6V(WCq+*F1kS+B4YghjOP)c$wQ_?7OER@_q8o zX*oA=E2&_JBgI<=LvQh`D9f3cEyNK0FmErUC%w6-3j!$NRiacN&R&6+b6%skfe*_r~|m9%QQ8ZgCU0z5^TJc|WtU(kR|l z*Ui#0yd|-*#v1uHn+r&(8=ug+u&%HVRp;9Z@KieFd0vtwU!dmTM4Q&))fj?jjh75? z_v-{L1=Wh+FKQRS=ToxPeh7PKp1;sW_$x4_r^pHEO7}pg)ZlsJmPFi$ULFw!WqTy3 zK`Fce!eI>y|FplS&kp3b?QPsEP4(-7<0yM5C2uwsp;!7iDO6tQozE%?BKJj;f~reZ zHPa^#r}p9J=-7lI&(St6Nym4OWv;s1h{y`AE{<6g5RIsm5#nx3&_qL{MTFq5)TJbY zb$u3SlZstx#IB8llGyIi@?!!Kd&e}ZC5k2~Qp*anSb0<0;_R3L0#0BhRnC~n#9l3( zSrSa(`t@6w|9!~LyZ5E!oH6z^4#)zC57ExWeI1%V#EMocJd2^&#d3F)E}vyvEwB*% z471n7efa!>I6lo(UDKfJB|AoW7d#5AC(N=W^mlb`xW`Yl6%#2&aQ~+Ap)%C5qPWQe z;p9A{D`abrYn3iS)2%0mQ(^OiAQOjG>QDHpL%g~?S*f$YOK&F^$9ts0L38a2p3rEe$e9GP2+=lEzPkebnu|8si@NJ{M zKmW>ohf#%QlUrtFqRK7+r>>tNj#i!!6!Qp-5YIQy$y6RgSoyU!c&M!swcd(gST5Ed z?h2>zoyMGpVt7mp24yfwhN0h1qy-lf=rY|z;aFy|-+@11eA>b<64%2q8_YjmiJ^vy zBs@$#+$h)>KIFtvxCYvIhN|#?p89A*05K^JAKJ@2aTcOYEYefeu$G(JC)iGfTdO$E zWW$|STFU;SP70!XrZsfDH27#kW0(4 zTH^*O)^d}Ca9cmDj5LZWy6bOE24d!n4npD)^!`pto~CsRrf)obxaI#TG~I{nIgny% zS?C3eptfqAWwa1vK<7F*+1xE*PZ0E8;Avse)k~4;3kXIN z40_<$z(hwwoj8#HS#YJQ`u!mo>5H`L`}3Co$~+6d2M{P38F|mCZWQ3Amm?*X!y;v> z=p&%PwyAeTp*5Npm{l{KN~l!Rl3iutRT9BZBp92&gFw79Rloz_u-qc*IGw2)udnEH z{l14~X7v-PgFQ}mmQ!a-6=6DRO{~xCr)5HxCU7SXUcLm}S{Jf95+{tn`gQpwPD~#? z%gkBfYv^HagOMw4#qIGUBz&+mDeeK8D^PmWh<&lLF}s`lhVmbv6Io0$A{j3dj&_|e zh?~N|GBC8`wZ)np$!TMOYF37|_qbZ`i&hX#EcN_YDUjr-8fi0rv!MdWi)T&UX})Z@Wc^GN zKsTdOn=;9+EU6Ya#g{bi{gG`e>&KZu;LL?E1$|eg`Ujj4^QCJgiyX`7GYpAc+@8%m zc61Fy6vuCTIE>s z*Azxr_XC{j$5cl&S(|CC0iV5n4uwX`FZv!pR?Z$u>~zNNvA_M`ZO+Z6R|bDb30*A4 zi%d>3(YxOy8{G{Zg3jPVPyYcJ)^5u8L=P} z+?@v3Awh=KwXQ))H5qXG@bMxCxis%(gP9gYB6!CGZi&{FH}T{~Wxc_P@cwivV1t#c z{)hfCLNRV7_@^Z?$|!~m8kf7m9ZroQ9-!R>@6Rrp?g3ZP;X|O{+ZcBawpWoA6-v&SSyue;G% zWr(O<-sgt$*56jk`=XIhbwm%Xb}C($VyRHlEsf*ys&53SUD!F~1An%=V@~tPWOY;p zL*1g>FIA`aJrUegP;2;jTHFg|fd0h$?(?il%NKB0JLKLJ zc>-dlf$|_xl@Shh@y?x$gd>P6dpo)mhC6_~hxz&YcgS7-Hj`r6+c6Db-A^79Nq=a5 zR_U7u7C?3je{nz;ELa;cJ2`cHtDt{YKusAX5XIorZ{!iqg`5DOfh!Eb6@?b~) zQ=D&byC+s07p%^H@?3#hdd+3|@!hQd$*=o=^4r1rPg(x+e&*}YZ2=~fp^^u_`HH5I zI0l-?@eRnK;2(grZR_PR2?7P+*t!3@{IvEOp#2Y!2l1GHz;rz=dh(+dd56*de($P; zpba4uZ4eK#;hU}CPY$8x#@v@LTmJyta{mA^N)x32wHW@_VtMetY5t#O8ZPYNp&pWz z?LWX9Lcn!7CRD(y&wsLO{GaR=5CL;hI*VpbLXDS0?;S!<^Y~T~mjyDh=&|g_!e@@U zC*H}PrWJ;@0_@KJ0J}QzR#&pQZN+@(RNzvnF&mGtk4=&{8=eL!Z7{d;;?G(uTb;sR zxc3BoIh#I^LWCou4~DRM>vO~qUG3#JcirZ zM&b5;8xUhKSNTikK$>kZaJdoo{!V^kqX)(%L-);^DRcmZhA0Hw(&jm$3w}C&UHk{Y zAIVjfh93v!u!>R^s?KxTF01A1>sf+t3EpfFerBh;O=xFI60Sbvv%#KE3Y}I2#0cID zf0AUC$s-HX?leJNV}eW)ytj6hDOiV~nI$oykS}U|e+8E6Bi;2+d~n8;CYoWUBd$)C zwZ+IjLfQdR)xj@3=)2Uf~dVCm!1ZauSPrZGHGzABg&#S2_q$yGk-ywV6|DPe|zXmlZ zk5|M0r1-C)g6o5DzM8d;5KjI?W_yvXUnTZN^f1o3>!Gs*@?C--y6#VrvJB2sS;#odLOF`0Y}u3Mhis!;FOCr zskJ-Rsd9mupKKyGPc|=@xmG<(;$E-0&EE#F|6t5W!|~P| zmG!_20CR2pSg|xLqhV5c5CHoxheWT=HX8s|Ee@agePBZDeJ*9cMzzj*Xh@xzFdJB) z=ghB+D}L6|RJMLF&q7VhEL^VMYh!xbKe&XaXuK6RSLNMEv3zTAcQ3*je%+`We^R}G zXpS=mPtKzAHm}-U2wgh&v_J)(&`GAS7IRCvT|i>`2=|xDxUlkk&O(|^pQ_b`S;maw z=r@QEIPYu1AW)-t|0GP!^#_RXQ!g$?lTyE+|G=`l47=%DE*mQdJ^S4E&gTE_lNvF1^;p_|7c5!LY* zvS+^fQcWp5DGj_qktt5uMc7cs`oY6TQe#~Jl@tuk8!*?qp=bfGnc&DuU6rvx?(n%1 zlOzMo*9IBuD`|_Iz_wUARH-I4GpNOqLW4iSHlRp#VFs<(gP-&#hBt`q5OA!Q(A`(b zB1o@QG4z;LCzWZI{GU*=tkSTm!H}crFnofJJGPSyD(DdS zan8Ctj^4)oxiHt}==?y-Y~{gqZ7bI53(lL6mq3FA6Ws0_|6G3RmJ_LI*7mf@^j3=^ z@ASyI8i~>uXvva2@;Dnf*Syb>(Bd^_Q#|C_1OlI|&{V=Q@vLar^%C+19_VQe19PB-nJ>FD)5)O!{vJRKeQc zG5P|3Ok{JmUw=_g^4!xTV=9bjsvs&UEdde*uSMl%3htDz*btoeoB8&5zsvxaN0vyr zUbVen)t1&u1e{@UzDiNM$~9dy+Q_jH#o5iGgK&cr26S8v*?s((@A<|qxZN9r2-$Tq&~8m4u>VC@r```#x{!G$Iq zSc4RG+BOu{L`yo}xlQUw`8rw+qVSsEt+rxSu(*X*)R--KV#u_Z(;1J*O!SkyOq8^F zF!}Mlkql50*ns>fDCC9FRo#Xu!pVv^mi}4R7rGl88#Fu}fAD+Wb#T#1Y7~`d<3?91 z0#R!s8=MzOl?{5lr1bC!?eVfsDE*YLp>>5-z0N;CFU89qsIf<=&Eh*(EZ=jnMlXrz z=NEN4kClkgAqn`XGp09fpTBKZ)s&s}(K77D>Ttk8nUND7K&)-T74=WIH6iRx&$aV%{H49#&w#I>lRmDMY)vz-wBCOJ9GSlXm_39^=W`d! z5>8b!E;Hcc;Ja!C2*59VFVPxLG#;SBQ=K6qubocIS^H39yD%zo&P6U8=KkLJ|Dl=sz>+8}O^=#9av4pk^gcGgDvVA5}3g4=; zw*PQhYona?r2WukXREHhs_J_yZ<^^`xF{gK=NWSF#_#f^_A+aZ8-p2?MvqHJVI2e< z=GxfGa9wEw|efa5B!D==ibIrLJ>@9*hG(c;AqBOmu4*;<{@+z z;vH3Tc}kc;j4!Fw9yYY86YkWd_uEc~PE3Q6bA^DRXBM*_ z#3HkYzL@3Ia9#=#e9h;kZ8!A!duOQ@Djw}*_T!v77TNk(N4%4>W1*2?ibBlR?ZH-h z+u{bKwS@X@{da2?K#JtM-a&(!I-@p9YdH_UG7jpB7$?qkeic80{kzRr zHuuSQ>+Iq0OVbo)8rK3Ek6P>2W#|yza?pdmuaEB1v7s8uPF7O4C%EoQnGKGV(aTy8 z_;x$9u5E3WpZF8DHL@T$d>1Y!-j-~+S!kKOD1lEp&z0@vw&wB5Mn#vG+(6bm;P%jnv&Z~)JE1jq!B`wKkPjPyBJ;=(!pbQ;NeO9A-3>@_+h+)g zo&J?oV|rg9p*&g()s$6s$8q*eN@Q5@4gl!JeNn=pw=ul=^1tJc97G3ZnC|wY1ERe* zEsw4fsY`;c9Q>kL>>R>`44`B}E&AWX+_5l%T*HeJTvG}SQzwV?2St5dX#JrM< zr93?a50T!Dyh&#PLl`BG;9VrxtHl2hqH$G{(n4rC8{}Y+ro`%7F5_5(B|<*MC~Tu2 zbmNtawC=4&$6o+ep}@`YhI z;ocXGhaT6*f*TS@=)O~zR$nYhgzWJ2>?SDe@mjK5xC>{Tz_Fj}%vxg%5?0!o!q29$ zXp?hkQrH79yz%FQA-EZt;2+U1^5>ySBm=h~o#T?nQJ#~EfaugJxE^2X# z{cfm1eyc;2hlE(+aK(WU%ZsLR@mj;;-gWSjKn2fI32H?ALFys(dxT4=Y{F`{fkGjw z>^(Bv=;lgBd@$?lgaH>HkJ|gH?1x>N-~hKPw;hLDq;xZ_VFsZkR&BZd8i%v^Z1p}K z|570SeYs+eCQmVF`S5OnY;cs834?V%DLyQo&YtGINSk-98F67DJX?H;Kf4QKcfwvl zZa~JE9~`tk#Cqs_$GISv2bXo(SM51mz0Sm&f_t$wzR32=!!li>c!VJ`7%F|+qJu!` z>Y5=@M;D6BSPaew?vS?8woo$YD?*Vj*iPa2rZbaum+DPQ~i z@NA{1yy4$i_8=go*cPw>ry;~Lgham{V{SW8>`GSwo;qk|;xH6IYARJxD8~Ls0rsnU ze>as-VYV>-s+fp?9@&c4S4!6U^+)kk3r!9GGOL{M;PwUobu1k1JYdK?`V&M4`p2vy zzs7V!J=gG3g;AEjnidmF|BqfVigf5-$RTf1-jRIi_n$qM^?R=|)F#kTgU^>|j9RXe zqEcPBt$-onl5nWiXY-1#sc2Z}s1hDPFF!Of+5n2JdE+u{EE$hgXkO8ieqJ%lp+We?xR1#43D&1x-2A zE-7Pz-J@_vBbRy}oCS4yhpDB>pzZ;dcKF?#L^>%Qe!t$0O@HQuKq4kP?m=DDbH`B$ z4}WdhJNH!3Q$v*HZ@ym`oS3_DJo5hM7EA_p*e?LAo=d3f-a7`%e?^ToRMa9dGJ68VRA$l-0pOIHxTM83g< ztyi&pD+LB$KYzbEPY*)}JbX=&O<~S>6kBwfGOw4lEq>mS;wJo(U?cMzc_*n{8Dm%N zJsgN@^@wpJoO{anNaO|IL$qE%^KZZap50-X$Tf0bM}2&1nu^PQ&gevL+mWJTi6Z0t z9g(A0{`PN(i1} z0X;zEf$WcU)dW)pKk!WA$00OcH{M&>9a51O!9Ixg0$UV-K!Akmi)J$`Np5MRmBtsV z5Wp-Vu^cBf&o8utigcQ0@NEXKQP*ctW1gekOW}<5b$Il;Emj0t@cu}<;vsg?J85iz zShvRa$2CfvK*O9T)5nq`L>grMZ7mI&uv*lOXEGq$J2=0UjRh7LZBz<|co3A#o7d2m zLvmFz@Z#50ReoGiJcPuX@;ZUkX;P}>f2KW%!slm8zBBcUfuD1Sz;=R0${wx=DQ{J( zJ7yj=L&}`r?=$B#*oG15KB`_I=wS38K&{iG#b@eli4wU%e&0bM3`oN6#|CS7RrR>q zj#pqHPtNs!#PGtGv=Z;r%O0kZ)=^`6qi3KGj~b-iUp2%7>@`NjfWr4 z`v<_q!v)+=SiTAR$T5HQEFV(?$J$$bIZrrYSSw2XuF|8HwF@Of3UxW#lqlmO@MF$V z+|8pCPm$Ta@qV}!5e~1hZ2OEhZfp}?uZtTlDKU6PaPcy+qF-W@b<58+n`K^?>5 zO~k0ocKF9)J;ARC1SpRnf+OKe{h`#)D&J}0$#bV)Rrcge-YZ|yz1`}Mrs(u8(xz4` znqfZ$PNS?`f;MR?Sx#_$B6)O{6@0hQX!Y#%eA|uOA4*KK7BZq_gt6Qt)n=KsV=;9{ z5(73kb9b~+AM9qJ`^3meUiaoleXDSAT}TL^ZsLlharO+0yfx#f>KHSWMYNoVnciB*O# zWz3|>>sUkOll3iyY_OSVm0$Agp@cIV?10t{7f5f|w{rPU&{rf)v}oB5pn9dnD__Fo zoM?hxiEU9`i>BGKoXdpqKxqhx=k>JeA<4I+FC_@)GEoP{FLQ5{eh*`!pO0*Ityh2@9Z{$(nP+RKri_)Cw)hV64ZAc3nabsuQT9Zs-G zOf0fuV@*inVQNT`PISkkt1BgWC?AqEQuwi}p4m(U|;D7GV7?)%;`>wTas ze_FJj9bF&mCcQG&LxP`^bU?R!z1|rqJ5}=5#YURH=jEpUqlRC2&*4>O_EHQdBPD== zQ?j>kfgELc`4J;7Dg4`Nf3L2KRCAJRPRvdQ;l*~Is|d7=$Q0{?<0XKz`t5UZoKZ-R z)z2a$Tc$V$XBVz*+0@D*`P!nx!6BRpcJ_z0=xe$mo#7YSIAvHphVWRIlJeLf(`v@= zZG32GgixAB9f>FGgiU`fM1m6*>khIwOdo(vm4}z4W?~PP(OpLJR_jyva-sDx;U~6r zzZP%iTX=2DM!%@~q&sfYGhdAdKQC##OjObj#3P^UkJdS^%L|cLW2ii7-w}R|NE5(y z!xF{g&wzF#8n8x;v+({}+&NOF*1757f;+SX>azYSWXUv;#%`j>#6iJ40=`0c+znKS zRLgKKIpK?Jm|G47t=kEDq`lsfWaRwt>Ulf9Mc^;Kara7p_BeF#yPm*?!cxbiD^({1 zzyXSd>%+Tdb-2=~1$E*J=J0s(KAVB#4H(Q?h>zHE*K?CdQ(Dmw(XgE^U^a%ACdcmr-7` zR|2VWG3UhRHohfM5#r&o2mb1wggS&53UU6jLttz*CRm)Rf$qXkmz3iTeVPy}a4Nm% zJR^}I8x(u<-HphEX4bIDdRAWov)c=}*2Zt|kAyY9Ue7*?B0{;c&KjPkAHb*3isN(~yAk>8{kCT$i#< zB&-8C9?Vaa4Mko<@QTAvLv%Yo-a+Y(f$()PWBo%j{snjb;=;QM%@27n0crM zElQEno?}W~?Lu+Dd1!pQ@qce(<56Amb6C~{SB(cj8wmCvwCo&NY*>juS7 z6q2n0C{vj^)7SNYVJWq#!oqyqM{rK8dX}+*Bq#oeZJJ5GtG-p?Oz!6tw>N)A`i$I% zKhzt%hzA``U5+l{pEAW38ya~X^6S=FnVk~bNg#N^>e)yeZQrOK`3f_+0bls%sEgOs=-MzN0br zW0nHFFfdZKoEwlM=1+cINrX`HyN1^%eG6R>Pv28;?-ZXkRi4_@XjC!7YGs$grCEUB z_Fiz~_tRS62So^3Hd4-9^g#+2woC*O15PI(g%h+MJ?Yl!@OCw8lzm3b)q3(t)ufEh2tK7=~MSS%EB!Se#wyn>NyTD`U`jrYEC%3{zf5<4(_A# z1yB991DzLqky1q%Q?0n74(6BGbwpj|s;@H8@w6eNY%G)y&DJCr4D^o9y4&`s(YliS z^?`k5o|-F|=kV5Wiv)hw^x#u_R9>wsQT=0&?s;H%ol*$r!HETxbP5C%Nfqny zmtPMLMBX6W6(LA^M=B+AFb^f+zg4!oI^<{T;Dw5qK;aQd6^Xd<>JA+k|48qJUdla| z8KEql*UL`eV=LJ-H!K@)ab_Ri-!mEaPSR?9PiXzUk&QW9h}KXwOq_iR7^4@hWlxNn zv2I5B-F)-;p`1x zHy1vkr)@-VbkA6?1pw%$qv<7lZrFb-Bqw%Onl%;EA|8_0R%G$0ksl1`l)hxUvE93! zZFWT3q|_2e`9IXS5Ea&ZLTU)~#C*Q2o#1Db^_JI~iY4`4$n+#WP$rZ<8>)*2WylUF z_bF5~U=91GV66o7;y85@*lWVr;h7l+M^3FCyEGMRpEZ;WOlW7`8 zlGIc+KnTs^?zd{YRH9`t_um=M@^i4^^y`_`ITb!`IMFtA67`o*1WqaOzW=VsGVLrL z=nOs(H%oXr*WpFpLH>qqwyyisaV+Qql1ivZXSco)?#JRjvu(;%j6J~OIhJ>Uj`k)L zS7LPnP=Jmo<9%@Od{0}*o@uRuEieV4H}Jl4UxuAW72LOF9k-9|a6fZ08iBHU zP$tRBlFdgRQzxgTAC@WyHSN7Pm}zMTfsoqq5JKt2X4TY26;V$gMvt+0?un-JBX$K8 zmm5lpC6VYHMrE0sS%+kmjinr!pcC)lWVx3*y5X9F(RTb&njVi`qQR8$qG zZO4OzaXx=ECRFPE#5UumaM6_da*lQ3b?w~QM!@yxyrIVcTN@m+Qmr%iVfKEK=(JLy z?X9RrSN1)h4#9QH++|=3YQGjw>d(W7?tHhQ4$_>_8lktdVaMCE^-_vPLLBgO z*Km!8I)KtdB9ASixGZbo2yLvh4aD|~sN`{5d;RroYyS>Yt#45G1Y^$+A)sNRVf54i z7y8l97U$fNavA1!wf?1}KWYSUa)8Lvu#m@7~%GrzmndrFVyx_e4 zA{{~e_&*%1*u`TQ1s$d*9n!xw642>o{Xz^3f zeO1|aaQ_wO)5ONQ)WwYmjHFkNGU9OAsMj}ych-E5Df1+6S$#0Wh2vvNfPh|8UVV)jc8p9zz) z_L<$*clqVYo`QGOH0AS-qd9b?#G@GyDYI|qn+;WjxdBMG1|ucQtJdE39Wl`+^jc^X z)>+&B{0B(`H^=f;l%bt<-dT>p{uow|yrx1C_Kr-B$v?*^2gyiigh-=rcm{Q|g7ptq zLBn{nJmxfPrs1Ea&Y(!~Q=*b{3R3YXU7fFE2}Op9jY8i*-f}jT9)gi)r|g%@PA?jZ74WT56sn`;PwpZ3^y#!K2Wqa zdlok(J4nXVeSTRI;#C-5P)(_}mH{HkH?G48RNzJNN1u-i&p%J{2vM z{Pi|1@C3o=;QH)>9G7_dB8}{t@x+hz=xiEli;NxKrm*rSL#0=)-!@m$sG~#a`^4oN z$lgMc9|WGcN}$o%HLcbKQgJV0-RGb}2Ss5v1;uC14*nk$9S6}Rk}|JlS(Md9bY9F`l{%QxrVsHcJkfY zosMtnqdleFTpN<@4nkXs4~j>gHMt!6?()oPu{)!C1023mRJ@kuxk`SgsZ9hK226u@ z<)v+$E+<-t8BkxyT}~%Fv$qE14ZQNK?DM3_^<)Tr(xv@INZ(mJAFItn${kKs1w`yP z>5;`J%=tEThS@1Q_QyyV>le95b3}ybf!%}jsnh=7j#g*G8&MmzhST4VTAjf}E^!$F zX#Ned+0r#$pR$MtrqFC-?D=fm1hOSr47Mhp`Q=>E7L@N^NrY34}s3I=kdJ}5)O zq>;ZvNC4)7ybzX%rVQvum?f!#b?wV{}iR^$u&PiH>WfM>Fc3 z6XG0YT%u!L%hxJ~QGCc_QVzg-+$ooyVu+y#WALJCFHhmg*pf{Qt#Rg8t<@&bt1oU^ z8z9+)Q8!NEfeF?`aBfTBzNIIq*Ox+Kcufi42KX%E*vu^WAuV9}i4<997h0_M-`H{; zs>IUngpbBwWR98AB1!`iHL2HXWJB--5(wcXrR%&r0`wQg`so&kR(Mm zOSM%eV?Hz4iy1t?Dojlig9|EFVYoYsmLJiwqyvIlK)l`NKD}G8P+6FQ(WujBER8zRi9wA)(v2CTQG0sEQHut zk3IV2%A`Pm#fqn0(5vr1qN8I2c`%%%-7VDRpYB= zmk?mPE`+WM(Mqa4lrbp>G+E!#L$_o?=T50VsoSssH#Sr=%p-dIYu%*E}ie45?|XL8vs z+VOgcBl#-(ctpyDypC*{|22d z>mLAN2W%OboPA-;`_*9|uZFQGV_%}1|D9G-u3r4vJjLatksg*}G?wU|mHB|}6K~nE z&W$w-hXe+MiruMtc~)762oc}LIrKm}&B_S7(d?>f%sH`#dgi$8o=RE&*IWa#u2y~G z!{|nFBzq|7t?moiQ&?YWC1b{0GY?I$Q#J2g8JEO{c81|gc{=PG1b zxzo(8h7p2-q{PnyicQG{<@HpnHnuBIZssA>v35aSZ0NJ))~y2rMQ65rPy~eN#|HYI zCW+2b(+6pscdSfd%lFU(&PNLLQhIad^KNdo%Jc6R3rnKyma_NyL5z*oDR$zrgAYy= zbSpQhnU#=gK0LTgnIYu4P= zRt6;TG1?(O3deC?;QU22hJXYm^r9z7UU#K{|0LDoh=Dno@fq=NO?Br;Iyu<@~ ziBWNaB&I8P8L`?CFZ0eagsd5qZsJ7Rv4E;Z%iw$xy~*FZer%)i@u98c6_QR}8C6*L zab|L5ux-3X5#V&X3*#W*NF_Zg&ao!Q_fhd%@YxBzxPO5F>nVtcw!|V(ElTsp5pqAU zj-qzp8$YS*)ZZ*kTI=0+x|6@(%v*xJy4|4x50kvAt~N!rbgo;f#T_}i^Ec``Sn!Y0 zs1vc?&s}rTi&a5z;$(?e9grUB1=|SAP*VW~U<-yleh+1_25PKlV`u@?_(_VZA2<2_-chlyU$pi7%YGC%1D)+3ki)Y4Z{ zVRrW9YM~pJ5ol&`@fWaJFIk`Djox&dThe~aZsM9HXzZs%(sDE8(Yz^ss{Z9ynVE^D zIf9w*@z*$gi1(Lc8qIb9!eL~XoTfb4fSF1iW4m!m=x07jk!mk`u9(#F{Vdh%;WE6v zF0?n!05=Bwd89<$EL}Dn#Nj~=^+E3-EkF8s#)8P55e?~C@muYn);%+6Y5qqTi`KFw z*SjO5<6HU+V#Mj{cA>w+XO4LCJqi16lIvwT-H*r5pfzw4jxsFU*jIWuZ8AHygsr(N z(#SyG?GWPHYGI&~wq`b6zw_k{FZmHv5cSEARHYEGH|1KdC(zoyiG zv$V*BuNwh%a!B&+ff>j9QXb3Dbv}oF%fsyydmy)l)*^QmRST2rf!?-_DYfp$?=p2D zp0YD6EMc&<5JR|)#ogVvJdRFFG44t4T@UsV$NYQ@{{Xv9;j2Ab60b>UnT?9kn@YKa zWC0?N?)ChtB|@ZkMXJzyGllVgfvmh__Gb3(<^gIPD#N#(*v@i2g>{|!QiVy=_+5u6 zMth^@pB8*G)_gq#mvEHSwD{DsO&aZzAUm-1_Zj>v=vPH_DCikjV5&Uv64|Uk;ZTbCmF?c)2j)qob#tB zyBsym=kB(LcHPb$vyL&J=T)vO=Y+@})_9sDwovGPL+Eli>)y1`hjP0yX33vv z3^nTup9sekX&iS}L1MT^U;$omgID~7Rxd#M!KPPY4x%_akE0CvwCiQskaY9&b}d)T!ry{>w+ zcQH!a1E-i2Gaz6}=hRmfq$_D6K@^W8F*3;{@3@jZN%kK~%dM89_Ih2N&^b69^IQ!2 z-mk19$EQx#*6x^HMBoFS35|Fzf7&=5`K?qR+SB^`%-@!!&xg^v!>zZ-ZP!$qD`UxZ zL_Iej;RNtIbL(5a9@8y!eI+cdEu?}aW^dj}y_}w`PpPY~QBAI1hE(9xzCT0=?k>0EvCj{{W^h{{WzxlIe40a6Mvi z{{W7faay50kCfuJk7}05;CHW3&^!fsHN1B(XC1Av0Euop`}1D&qb{uTRcNh|=TU@< z?kFDh^fr^=zYJ;?;v1PQZsf)rX81Hme;R8Ke&!O+roI&w z@-l{^Sp2s6{41w)()8%Zl@ngvgUi3m5c&?Lv{+Tt!RNS+kMJ|}txH`;CryTUi5M=2 zXvevzH62p=^lXx1k&cnZHmLjsQo_?jPd&xWS-bp^W_bisG@6#77?hl{MtsL7-nrI~ z29K-K>_g7+tyxarBUVjz)-Elq+C%;MoB9gHW#+5dQ(X>u=2*$^QT^tP*NJsov_xsb zA`B{`;{vje6-gzr%PNzf#M?Ek^v@7#P^{2e-`qsq9TkW*?0yIm+xT`g1cgLU27A|u zTQ-y`N-1cML0}S$TY9sB)AWsIXp%dPKKdDuV|j$(g?iz-mm}tWc1T`w2eWXr7%ciQ^^a!t(fnwt?ea>8+q;^ z51H8y3USh-`(E+7SNrYvocM^+rL?YKOJi|5KG@IVaauZ!-Q~JvEyR#2ha>{vspxkO z-Kl?R)RWOA>vNtCQ}`9mYVykSO0qXD-zyw#9OK_LcJox4>fFG#ax_j(T^Dc+D=6mG zRYv7zyBO50;O!-;fclV@-DJMCm;rz#2zH-J+SBeL)a)j)xKr|e?bMD(r7!K-!lG3n z?!KhLx|ZnXX1{{!GZb;{$K5p|cxLcTl~d&w+LGhtKT7r9GK!BglCj@U4^n>$G8xTx z2f?-m!cR8)W$|3c#t@%+=UJW^yD1vH9aYfan38MJ{6Fx?z;hJxC((_1-;RD5*+-)NtHbujJBww@!WglY zd-eV*)y+lw#wjUmd2)D)@C~!9H!JBZVE(m;?TWi5ON#1^r@YW_zvJr8Ye_pi54nFM zOQm=}Rn~2V#3-o~eZv0$#q^};#jT9pICdBNI`&A2(lr=vQ1%B7Kj0>@wOum*07Aa< zF43Zta;y$AJwd2gS}EPd6YQYJUTB8O=l&-es>LB0Dy_MyNIXd}3m&I!2B&U1Ez0GW z{nzR%U6;CzBM2dpj|~OqQPe{Yj=05diL{+(PcZ1V@TK7A?3V;9pZ6Q1Rx+zqKZNe3 zhQmpF)h+U63oUNzPi*O$%y)5*`DJi6x942WppyNip}4hpCPRiog;-Taw7$|Jt{#+s zhb#Q7dQF#%G@VI>wy|n$E*p*gGxR+Td9)E;GWlx^lOg^W9945um&A-zYDeK`b*y|Y zn^=_H=-Px+sTtVg=KVU>ABVh0sCah$BQGd`up1cihx{u@(}P~?u@an<_tBj$i>mm3 zQ|Cz@Q1i5&4SKhUbeMc+;kh-N0I8_kN4nua;%NJy_lNSXdboG?mC@gZ%q2~J?X>+( zKN{J8Xj(%QcksnBwnP%ls2@G~?_No$Y7^?0vRzJdBb~>t2hygF8Piv}$yU0SmN#e7 z?;z8@)O@Fhl`OIX6@dc;_B_^Aj=gY<}#Z@IkGdXStGbwTmxhcN|~5 zf!Bg?Mh_L!_&)9paj&Ajnn8U$f$}){7w)QLkTKtma%(x%aNQRODeQ5vNx85_JuAM} zybRO0n@@EQo>k5_d*+;R5SNfTGe)1o>E@G5xPjyk94*Z^sa1Di(JW@&}}|ej{gAAP%DJ$3S^O&bs}jH4yvSJ{dlAVn4}xGXN|c0 zfBjU3asL1iHF-3cbV88BAd^u-M5(xZxW+yHW6!Xw9yajZ*NI`$qqKOF?AMDljU!{^ z?~hYmIClkpZpJlKovd_Z+zY~fViRziEvJDGCf{*kD%;8cwYFQPWbVvyIuU zVPLk%9l$dY$@KOWcD-tJp-p?J^Jh|1da$CjASj>)Ng_m>+lfmSZNQx4<~bvgTy@}@_pEP9ZaW;i znFwiRx0+RoOostM+uv&EJaN~bK~d^j(#0IDWeZ?F6-e*?73QZK+jG!LZ7zc&(=B6D zCF0FA#fyZJStJpHco@kT9=P?cGgtAvw*F+-1ImvW+ZIVD(;4URt6tJi-9vCuSCce6 zSq_H!b6LrzTga%4Lb{a^@r}ctI`C_oHi_{KlTB5-K>?0?x#UKSq8x!5gZGI&&H&q* z))1Us=Gr1sgI0>WF<(m5G_;v4THWm~eq#=x`HnJwj1$yjj1&B;cKhLk*Ef(wJ?5i# zXA#21Ec=yxv(#`Z)mYy0mHt|XBe%M(^U(3#Q%`#frb9Kn%2 zjGd9+Xqrn{=@u|t>P4jGObmz2jq*W^FR=uF^{Q_gYPu`yurwDo)3`!qxBJFPNZNw~ zqhYxxrqStK{F1AAGk<8QB@}cKS?bnOrL<`5?t^rD_*Fdf>C@{?icx1YNi_1!GQhIk z8DBnQoB&UL#~)E#(t~?yZyKG>#@63d5XK&8RD{6_2_*OD{{XFA_x7yBf+X^@$vx@E z4>!3ItrT^?W+SHFySZMZj%vP-seQCE%QQv5gboS+02*$Cj^wJzmZoNc)=6*+M;kv* z2?I4lUesevCh9wRQKo2}ep8*lDaopZ88(F~sUyKPj~8F~nf%>DP?&?#DBPm2uocD; zEI1z3=*lv0;yI&I30cJG?P9pKm2NC`38Ij4GROIf$kUSc`tmzlM0c7-!m8Br)xM0r zdJlM>%i&*&`i-DlEma-1F^1 z#^&J%nZh8##;b*2ps!cfJUMChLp7q|hXr?JFhAClY-z>lcpc5LLO~eCcK2Ftn?1e7 zpb|xTxo2aIfiuHo`|(jJq-dbo<`*#Fn$Xu_w9{_V6u*Y$UOnDqK#&inD;d^`H-;W^ z=yUSKSE!~p+HMa|Eq|Rutc~M=t1g5R3|0GmCFPnm!FO}=)>6eqE8OaaQ&sL$hfTY< zy|=cG+E8q0_XLwq%p-2txn{0LQ7Suv=;}*j(LS+M9SoDk!)nHaX5p4(7aB(vo*J!)BB! z$-1%SOS{mjo*|Z3meK&xdF{EUW5i>=J*v&1RuFx+%1e0EugI8L6lVbb9crfG9TNV% z4Xb9M~sG>0_z1t7NWPFiRkuA64!$GgDqoqTSC9r*`p| z+CsWW%3lXO`tT{b%afHwsV>U@0C~?mR3k;glw5qy+r@3DSldG;lz{IA30?}Y83*wk zeQTMEU%T-Q?v-T{ZnwE|(5ZZ7P#>7%w>`~kE4Zeh(^mfghB($8xO2|w8tda5l90_6 zrK>bSW7H5nHv&J0jz~YKHCtA;zp|c0wX}{qpuS{#!m@_@rLuctw|rKQ7gjS|(&c`C z@OS)6rGbq|b17f*9+KTjVK|CeQqo2;Eu>fuxg3RVyK&TGHODro;tNZeEF4X!NaxOT z5)@Jo{>nW~VyRV{v?a|Q^w8S077-{yElHX(=JDeYLIBxT7HXi_e~RtV+J)j$-oEoQLs3x1e;+bZ)c-bUY z1giGIBcVNwP8m%~@{_jfx840RD8mtdXqPhVV_51^!32I|pE($Ua9e3p?N{{OHfZf( zxUh~>A`HnqAQDiq4T1j1^y}8No*hbD-;va^%5@#xMy2QdT)E-D6x-?g@)<7GZEnK} z0emVj=yCl$>l4JfWO|;5CbI%0KWAWKeW6V0oD=wPe>&FzCr(YRJx@wFPFQ;!M4Yd; zb^Uf{v#syiucMCM>Ic~l;+ex1_Z@bT+MIIsV=|l0yzS` zh~iY_xt+dt{Cb?2jJeT+aQByVulKH>iBsY?!z;^;NBdX9*68bSNw~LG17m-AeUEYL zUXiVMj^9hOjgjLrTmWFtQ=e}2t!zy?5!P1xPAU{C)s>WQZ;{6McT-IlMYOlpY~;3) z6JUM0MkCPGSsL#4=}p$5ZF2yVb48NBfb_2~`)*0wPsJX@a=J}kZ|?s9f{qX3XNfH} z?NSu8hCv;&m1!9W$vvyN*EMUaYe}WCv$UEuak>^cJ3EEXdft|E&n`}Sb)n46Ar2?QL1J5)id zM$D2sEf>Y{F^<>7F&n#e8&c>D%?SSh*@v}p`>8#58q;IbwadiQql8#mo2af=bQ8id zjy}DQwQ{}^@k{7eVhg47w8>ldksfv*@2{c$b(^N1$3$zjqcwa#;XQA~ej!U|h(fG4 zyD|HU4^v+2;2k~-9S&_)ONeHY?Tn2wACyKu=pNYT@ve$dGm znkJo~4h*ZyCviz@FqR-U63RKvWhip)QkBQa408n%#;z3Z0ft6t7Gx$s#G#CU+qmn{ z{xq!7S44n&R1@CC3jC+cVmJzTrlBXYCCMVjD-uh~rziKNLF~L$`xZ~MM#aN4cC$#o zc_U^S^#Gd8hW18{;FyH}IbJTi*tjnS(N2NCp%xGluQ%O)tXT zBL4tI@a@&L{ky>C>6ul=Ism8nRpTwsbWR@pwr8g3nDb`!+CQB^rc5HdP)-%b)BP#n zrlU5WQDUOpQ(X>%nIy{)Qro#8aHRhLD)lar=Z&6MNZiE}Ng!N>9B11#TH@Ej2B!(MfB!mYia}kV|*wtAK_dmqkl=BX+aCV_uhdr8_9ca?eV)$VT}t#jE6KZR zrf?g30tW;5Ny+K&Q5+>^ttMN@i4i0$r;<0=odo9n3yk{G>M0e;D zj({m&{ghQzQc#O;aY3ghk=0z6P?5xhmtY)$(SEfuAW_qja!xv*TEN$00Ss}S$r_i z!C-z^rjU#bAWg!c=kB_39Q;^tK8XPp0ym;(D{rbR&^a4)~Q93+8rE^ZzlBo zN3CMtO?Bn8kd4v~K*m7)D)~s}H!k%Ck_lsmE?EkxC%G7}IG*z z3z?kF#k`Qt(9ao35bb3np|3^MHV}S%dV)Esl|6LWwNp(^ z{{V)b5{ko4O;ULCb#l2(kGP$`I`lh4NPM`}aDW9+2SM7U%Gx57da~3OoDM5q+7MSY zHwMfNXHLEQGF|D8iG6BJkq0LIlmXX77~$LIZS}71&iNy^c_3s)XvyAvNb6oDOl(!* z6z%4E5tO3SvN(NH!TP1WpZ1N>w}iX#5PzTY917|$B6+^esL1|o0uWn&O~Zr7){@0e z2_vqZq~Ek_PK!g4(zILnCx=mV%n`^7W3L@*c{S#+)s{=BkJ)5k4Zh>@f4n_SYd2n{ zH)Ve$Q*`61Jyr>$xYO0`9SKF=oBmv}^{hMVq`I@TF~TEKpC!1?@2&vOKDFnl+efkB zVqsG4Qj2fZetQ|$cNa6H-g!?lGI1NJlXs~bbI+w@uZeAKWtof;DGI_lHv^rckOx&H z@snLtu+1)Hk1nPM6A$83y8HhCBCe&W+S;o9o$lhkl~PFKko=PT*emVVk4#oog^jn0 z1h-eWaKOzz8>oSSFnHmH^EUy}qw2q(?(0gRl zr9x46bGPDo)G(O1RZ66+y?@N>G)99#x`$Yb(YMDNq(P*_cpzl)>(`&csw^paXUwwP zX~SwbmP`>Z1RM^ajt^?aSLW5bekZe9g*nIB$m%rz02|u)iuIU<5{SYC86&C74_>v% z_<^m~?$+YU%-dZ@8FIh?tO3Rs9b3I6Mi`XbucvmI7)lst4oXj3?|bX=Jpxv@yoHl4 z0|JGRL9YMj-&9!*zW$?BT*S`po?){~^eWU@986gxibhX*Hjzvok0>IrYB%N^V{xp4D4 zOuxHX`fb7Pa0UlF=Al!Xsm&cgi`()S;sc#h{y((OgUfVm18U!Wa9 z^d_s>*lOlA)O5r|hUo|pqL&19+BXk+Q>RXKF4E@gZ%YG%td&^bUhdy{-s}2pg_eYF z6{Sm8-1`j4G+~&60AKE$@JFR{_L@_8n?|2UzOrpLOEM($j7%I#cpH?TyN(pAUO~S2M(7Xe;|EwI|Wu&&$ZcjxBP-#Rgvx%Pg0X zoJ8^*sUPh2$F+JstFQQL#84%@{j)XdzQN=vU56d=GmvY%NNW z`02mUW{arWT)}o9N!`6ioFQG$CtS8V^~VCS^vG@ua)*v+RY?}^SP5NofyHst$4ZB} zB%1jfR-sC5IOzRf`RGeLUKwpF8+|%6E$5m%+lE&r)aU+MdJOT3pHI4;*3MfEJtNa+ zciPQ$4-+xuhL8i#*1F+|RcZ3ZHu*2k{{WDtr%I$p$4C8kNaA#3 zCGVvDFZ#Fm98ZY!JFg7ubLqEEk{Iw(R5@2u$Tc>*9G7?PD=}acCJxC4nbhK{!ATxw@}X4#=ArUe49?+IKqnrjpsusbH}pMf`p)jo)(C#q zttXa{v-&z^LT52mve)Vpqym;4D;z(IZfzjqZO$qmsIr&+pRL^%r+O7c5)y)?NB>J!#T?2 z5~N_{d(&&>Yjfu)3GDf%~ z9_k4P2a(&-lf$_DBUO{!^}!uMW2k>K8FC-TRle@A0mP;;B8+i^J4gGjeHG-XO&b4C4p~6_x#} zcW?VNd;b8vn%C=8yP5r7r(tbvTT;1wLggim7zp8k`h^+EVc3#?ASxNPE15>n!2$l@ z{GCZg*0Z_=;WM0@kh^Y#Liry(lL}uH2oX?4MJNP^|Hk;xz zCa0v$2@&2~+l89iWyh9eKXih6p1o_YvW_WkVuE)%+!+YBuw1vWnh$33DuKhTMDlW80;5Dv0NBXN;usBJB)z@xkp< z-`Pv2UrlK?H^kT^9uK`)I+8~_OLBKMM5}?jvF-Z*06Ln}O)i_=zjy$3jE)HR{{TJd zT_TcYt9fO%kqMI$Dcu~4`hGOl0!YHKBp#XSJwFlBkw;@^;#Qs^=7kE~N{kSDkEL;O zT^nLqxf#Q#{S7ojJ0IW_CUgoOZ}!><^u zu-Q31iKAo-R&~D*9nXt2n_1klBfF?4?zfgjV`v(X()>lI-!a0)I+p(cbnt&eSW%mI zi#st8jBO-+7kmU!3^q9HO79f#s<95FagO4fAu{%tt1aH2BoWOTUc^Ej-!M{s>l}Xx z0P|jP;yq?fF531hXv9Bh2HqEp^Rc;+y$Hg9JLQdX)U2YKJ8<|pR+3RH-0S*xhe z1OEWTve}to^BW5*{{R$+)RUZhSCYvvR@%RGdegV6uyu3wt$4G8nYPBH_R;_aX0Es+1Z*yv~+n|DR9s|e5 z)7X2}okLR=Hd9?n*^L3-GmfIX$W(HuxitNKOem_6(Q9+bKesQm$Dc{NW_VlWMq|i6 zzfQF_t#M{!&a5pi=W>l%O7VC8yPJL<(ylyBt-+`wgwm}U zQ*?s@ZKM0W&U%x@ca~xe8$gX+ytv_n79;7;>t20KT$*(0Eg#;_DzzmkazN5sLCwN0AdD=fUjw+>HnT2RG0OO|@C#d;kI5{ma_spnyClTm{H!qLT` zCfEHDa%CzroxOSPeT{k-fhB7#Yf;kWZOEvN86%y&IsCC?Yn zYqB+?IJZmUvFJ}J!_W7$(z{>Qh}+jZ8EtJOkl5;yTiYBH1f)Ngk5BIp#1T_}Eelofz0Q*>DR$;5B9A5~A9Ux_J%1YM!s2O4H>sum^CdzQs`JvO&L2HFb+C%>s&UQE5|ssc)qw9v?VO+L?4XRt?(ONU(i!(x(qXSHuil~OTrg!k!V2`nMHzPOGJFkRX?+DKv5PXuK3^~YZI*?5-b zO+QE9YRZ`k1H_?n57N3Ji>RL@diVbThdXf7l%T1-TIyqJa;}x9wZqA@N3|C$p!yG# zay=`W*X$NQ4qsfQ%UnA{x%Ui}AcM!Z_)$uXDpiV;+~`x}i+kJ1#MXRQb9rvE+iDj9 z$=f2c9y#bcA75(bygwzm*5P^7xDp3dAQdf<_krpw*_Ij{{V9yYyJXN z@YUkuS4Y)7Kd+HhZy=qgks24&oi@DOu3NSV84d5y^sG2E9}x|I?RPe1U`cneiQ$*dU79DuZPO`W7M!!F!sKEKkrL*l!$ zkvn;-AjwkQKdpL8+AWo=ZOb02{{VTRt!fG4i))KpT{PTYArjlqESYp8s;{vb;;<~W zm9W1u+uI<{Gq~r!HLP7oyRN%~P=5-wy-GeK@bS^&vs;HBZJskEv4g^M!1~v%-f0l( zY7*kwDd7VkILRMF+P$}g(Q-4Eo?x#1s=4!x{i+C?2)WKfW079;J|1{{vb)){9;H0f zxALo}Myh+6SFeVsv`+s3*O~K*vBy1YTg5g~Y5pkGZRL$4b)AmH&Hy}&XWF*5r<*9) z%LJAf>svk%(i>dyR-Ir6$nyq}_6WSO{DlczhV7~AzAX%O3k2P^NUfpX4}74j-Y9`D zu9H!Z{ak}+K7;swN-8U}Y8ppV#%`xKGJ*gIqiDd!?{n^IYa3|hBh8hOS+@MEpU={^ zZZJm@i<8i#Vsg8-oxZr@yL~&u@yi0mbvD!lHqH(RJ#+M~Y2zuu>U2X1Qfp(FwYM@v zBwX#z9G-FS^siL5(sc0@aK&#BiE`UdAd}SBH};fVtdYOeB%^lEH-LoeTDp~#P)^GV zyPmA&cCX+EYVL(qC3A{Y+)j_e#`R~8)Uay_gZYt;K?GptxZ7=K>=Tsp+6g56qJx=pT(Jb^!1+ub7Sc4ZO+`BU#tV!xkRBsk(x6*xw zLby#sQILF#8nKQ)_T+3}DC_Ea4ozU;C#yD7irCpH?N$;d*rCWQ0LcRYp7jleJ<{huYe>!KBK?HC`XDUhTQmD~yZC1?6e)&Gfnx9}- zwJc!LMMqTRsIytr%WjM8Iq0320cFe-9D`p}h? zkT-qG^MxSBbAm8%aZH*)D-j-Z-1=1{$!zi87~0s`c!pRk6fA0sy#dHp#~^;0_suuO zS{&LYs#V?PihRi&tT}FZ2iza#D`@OxEy%1SE+ca1C^iB859v^8i6+TeK_K7{tql$= zu{5I+uSQT$6;}TMNlf3-@qOyT~|T-EYbscWSAk2H=yW$4u|ls9nrOWjT=#l_SvxLc^!!E54BtOmzfLP zen+Bs@>E?$6A%G>jPb`Mj(?D>&k^bt`qhP|p={h7q>6EsJ$e0W%&7;@X(Q;e+Og!9 zD>;^YoSJUZ&$U4$oB%u3XpWb5$b9#LWR-a;b5i+evjK9BP#>LC;i}VC)m-PP8O7ah zk5ALSB3LewX<=t+G`F%JodDpsZ*x=lE5a7Gnh4a^G4}h!FB(YSG;trg0rfpkO7U?| z4N3AvUH1OBDxH66Q<|bz`TqdJ2-8n_Z*_62BytOIJjizdgV_P$jys>As&nXij5@8d z%co0m8ub!_Q|rZKxg2?~Yj)H6{7+TFnpcBQ`t>M8-)p>f({%KWBaG%WBjp_lIpCan zR?BEI&35vnP}{=T+=-m=*k`fhBNXL{q@$$2HaNJbrMLC!LcSxk)O45A$D3(*A|w`Z z<-R=$>6*>gw1FEF1WP6D$?~Ft6~^Z7eSjRAs$RmYPAbiP%jRJ}XHWN+CDWHkd$V?J z&f=wW5hXG@?G>k`cyV-HFUyhLu4DkqDrc{tK9!w%^smVYHobHvpDgu~vnc9%zO@~J zTN`0*ZWAoWm*>Ad4L(cvD)2L-LXCwgIph5Pm84@BT3o5$?=h>1p-Oy}9iQL-01Qy@ zMyl}mnPs?fgA*e&bY9?}Z1?x7M*OAJsb)_YGT?>C&rjC6YsJ^5m8@MFl(8{zPea%t z@dPqntV=XbR{#_{l$>-1yq`^y;pRtY5HFzPnGr>A(f z7^c{&+vCROj5jf#>yi0ZUY~NSa<=#KiKmnwx^leq1lKI8I7^-}R_kx;@+O2Og;!4h z0N3@o$*u5{Qij?m9#+k)Rd87KIra3anyhhYJ|=d7^9o2uuH5#`ck-&Vd7!mD3`0|< zr5B^r>Y_zhSPlhpx{r#Eo)#$W%#un#UCsbff4n-6%DJP0uT|bC@#l-92WKKZD@C`L z;#`qV3tK<>K|nY>XP<4=0?DW)!(L4@Ez7P#MhdWu9x?|{UTe{pwe|{+Hmk4bX@;Xt zx|K+EE%=dQi&wk0gt3ic0u9gds;LK$emZ;6DX>;Safwg>O2!E!oMSof)KyS}ryqM; zekDr>Lg}Rc0Isj0qv5S!U1`%;&aEzajxqB5qpu{7%A&cEqP$eFur|vO!UT*v?)T?_ zeJh$&Y1a4BvpXtJn$ow={ut@BYwI5n*ybCB%FS=o+l3)bUn2>UR^wmBU-HI?B^S3cGmv@3v41g z?JnFn^b~v$A)2vn|$)RK^l^yB&pdyL||HJbLdb%5l=H?W!yjO z6CFkm-UJSnXHeARw}p!^;Aa)nPZ?EcmHz;*>tkPfwA893-}Ul;GVYnKOC$=ncUo$r zWl~m`%5^-B0XYmk#yt&rr}inJJ~vp&w|A)NQHqBvjH9Bz_0Z+1 zK~rm)w(o6!;n`K?jc_souX9sdco$x{8}2Nf<=P!Yvh7W$p#XBj@~)`i9j@7vs|4-( zpf{G%BsMx-w3qkq^RaYW&kM9;B=Sx=n(LcPvO2xri+n){TS;cy zVPKJLd&V3o+58~mIR2enRHX0FRh%OfuXZ=VIswphmAaf2)irqb#<=DSU*d0K-3QjU zBAVgmn)gt^zPKQ;U%MKQyBhls)Yi>3v^k=bo!PHATZ>18bjKdml$T8$*(BhaiyPeW zb0d$s#ADv3@=&o{@=a}_=jvF8U%a=9IKyzQoRgF3iqDb|W;;*Fy-h=>2f8g(n%SNH zkK&mwP&A%YVMYLB#(isoY{P*WWp1oPbRM5t<(>|k_;fm9i<Q)WNBiF(ea!J3h5dc%pEp7kg44h{yt=$rS63|@ZGZL`kg*|K zZtj5RrE1CJ3yZs*?-L@+seQ9@$G7+b9&&#wo5_`Eo|j_Y9Pq?1Yogfztl5ypNiIin z?UL2O#j360Xy6RE<0pb@951pOB;iWSlcby*LY+o4YhE@lEkh^ zILQRl_Do#oY9b5)QV2Z=uO;xGkB5hR7P->xIx&oQ& zMkN@@11AF~uX^(j5%|5Xwa*LRwwVqs?u!JAP=;{(fEf8t%6a}Q56Xo}hqOI6%~jwl zH@L4qZxqbG5hN1W>NeK$Y03q^mFF-ai7o_XhTrywr4F5t)Xu9=wx0G?W>Bc33cIuT z)hPT?q+HG9uD1+Q{{YW9+sSXqz^S!|a!>5-isFR;DlyW`2t5Li?#|6JCq|y^joWvT2 zrv<6EQj-JF$Tt=B>s=b?kifu5ha8#wKJu*?g-?30ev z(!116*=S~8YpC-6;zv+8Cmm}D>C;RPkaunCT3I7Ivrgk&n5b2dkN0@ZVashKR|vb+ ziw-uQ^NMysb~9hys@ug2K4vn59(n%&c>OW`DX5Y?)YlH`8_n|L3(v?8Z>j6_q}{eD zM*Erim`yKEjwm={c8v3%T4tudY}S0N{Fc$=`V;yDZBrD5G0H+R-kYIhP%aG_v?#T#S& zWZO^UC5_pd*Z?ydd5V6Z^sk$(kyLHVo9{a*;%LcQEcRBG zWdOqGkGwP5yt>QBy2H$+p|oOyELd#F@89`zT+b@RL1@?BZys4~sp_pFI{xud$Ez9AowOj*@@UbBNb(6qi(%5R6zU%Kro0+!!MTMeltX48KYB}Wp06OQbyh&qm z<95DcVYgdWi8l~3*%;@Ez8#10FX<=O318#B(zP&p7CN zRx!m^hhOXDbIz6B-}(M#Irx92J-f#R+p3e7LgWnn4r_QgY?66BO=JG03@mle4+#k33i*`q`L(}7tmULowvB;`?Z z%TtfO@MyWXv9_|fnGBQdFCpAe08U3vc&@4mq19zJws&DJ6+?+5Ic4Mw*F`K|BAwiw zuhz#Lury+|k1FnE-*`6aVHg(4=E4Z`V0;(Ps3d3c&3ZoN8zOlCkYleEYZFy$&MZBs zK4`1@-JVHrrr&87u*;}u_fth16(BA~F~=3@TG*4w{mZZ*f_9!UP2*)wTE!ls#LE{4 zN?z)HosKRGyB9+RhMg^=#M{~_BVoJPae#VtuFk>P>{+CWHefIxZ#k^$T=W*zN4l16#tlwY9a}&s8g`9*$I2%Fz ze;VkeI0W&{x&57}N~nd_NYzLO_>_~M z)YVHbbr-aP*RSYh8nE`h`RI0fe7d?!##Ir-GWqJ+=qu0d{9CEp`Qp-2i!udsxAFe~ zCX}!hlJ_i*rsCS=k?6^!+KXWam9WUfv!U8}-~)>DXlB$eT(r7_L@-Y0k+6H?k6NGX z>W$sh)Z3lM-n;(3XKQbwH-~&gZUo5I)5Pi0n_WT5@J}3P z9@XyQ>CIE+l#}gcW99403a>O%(|hZ4h0--I5Nj;f=`%N(BFQDlJi*UGIu4yHQ(N)Y zhoJ@%PZQjpOi&HZPrg5{DaQ#fwI~dn5lI4{-ORWf zy^4%r{&}wt*1jO?ULcX6)6m&K++`2%7<;#G!xgM)#+$!0I4a_uRc>8-?0UzFek5r= z8#3Ejq<0tRYYTLk}-`rNnNwK z&@_!lQoR2Fgong(88ruA^ms9awogDiAM$ywz7|)69Q72`%l1ham$8AIjQfx9t9SP0 zmvWwcMOv8>rgIW*jfsi+bsa?)3x^?r9QLZZWTe)|mg(14-cu~VnHe#-AG^>00IsM< z6Dc?uC)TCH?&d0XRyX6au_!hpI`$=r6`U^Ria#w*JCl#(DcO^9-Sk$ls(}=Xp6k-9 zMSU^@^OpyzxC9y!3y`F+@}=EZ)UIjl@J!!5v}zAuFDieYDcCM(+h{Uh+5M*0bbm!* zw4bGB!#(tN!rn6L9zno3!Kr-L(T!Tgp{CCs8dxSo9l{zVztp#@$2^hkUARdA{=Cts zXkE33PkDy+V<%wa0;6qS>Uj2CMqFTg*iu3KdWu$MXlhL*wYDR=w~V85rFs1-8+mQ@ zAhy>K${cWy8jug+%}uSi{R&q)q-ysN$1@0)G-zUQSOy1>IO8MUuiR;IS;xLunpmHW z*~1TPe_BhH_d~p`W&txA%;i&$T!P#mEA1@gWA}SwADxOk-IA|HNG7n& zZm9ZfQ_Z&xHO!A8C)WhvepO9ZYYi74HS(4ch&u~Wv%LH&e z$slLk(r&;-pm<|g@f^&WW}0SIKRla2MLzD_XZ5eBtn4kc`zWmJ;YcmwU8x*n0Yzr! z`V8>5_&MTVD|mYX`2PS_Cg=YELB)IKJ*jg@dLJrk{vFgb3$G(o)7lp!Anp6Z)kxvL z3i@AP(CoZHq|K*ZpxGe#A0ISt*_Y6d{Lcb$8Ku*wOPdS1?WP0ek{Jq+54i12 zxU#yjzqqrS7xN&HqkN=t2^k*TXYs28Za)?4CsB_2@_02lBZS+9zl2DpGIBxn=e=XZ zue+G!{{X%XF&$lxh_r@PH+~(G#^4TabW5g#C)dkAFX35M*3U5891@ntWHl|NxYBQ9 ztc%9)X1alcQ#Y`~YLZ%}qiPaqB`=0T$Uq=3ImbEnto<`XiPlEcA#k|kJOT9e{{ZV( zHC#;^d+X?QQp44r+8r(Li|zGK3naF80?SdA(Hb-bijFssanl6g_2ROUZ5rKLB~<6T zcI}MTOA{O3F>KT7(zdcl@2z37c+gwCGKO4-TxErMtTPxc*`k?_4pnz&ihRlHiREi^ z(yuJ-V~cgnk;a~~Fi~6t_k&VszSnUy&Vy>BaH>54{9-P{wxqrpdfnHs}k;Wn!W3dG3Sm@CX|4{1C(Dtf3McI=J5EIDJ{II z0AYf59E!DKUCgm6BsTJ<1W371Jqqzy;K^+iwl@P4TfW9q+nj$Yo>J7bGrrj@k=@Mi znIsAc_RT^w3(FDj$K+~^?lM@!dvJ4=%`vmyn2Wk?;IT7Gv4)(c*vUV0Jxx>ljnv?e z(ttJe3t1(-H#>72c^UrzXo~2(BO0!YER4J~#F9rC01D1^6kUyBNzGi2ZNM?$e69SA zPpjMcNf1)VB#@weyx@bJd*D-5BGtw0K9t6Z1t2vgrb?Y@7LzR!Ioi4I zMM&_pmfmU%gyHeq(ETe&>|s$_MJ18qY^zhYbAwF~U9{CEwvb5_uM__Oc@8RrMpx-h z%r13tc!DdC1^t{?AN18fcPHjJKb3Nd_&soH98=vI#;oJ;$n6#l9`t{s>IT`*@LeVc z{q#7lNoSFJ=AqK2`Wq}=Wz_2%!xzMdU9!Ya`p5`zk6tUBx74M#aM4L5P3nwC@~L!U z_GL#IH172`zRh{3+^Qt+76=>3XvhBmTBEn|j;9z3*J5Sl=`Rm9ZX*mm@eCk z0LRuj3jTx|=NDG+{Lz$_!tN~hM_0$^T<-p zK2G^d!+jQP2>^kXh{>zT&T?%3g6oDeF( zxA5~th_!`TGu7f?F;KA0KZNUfwJCkOdN23}V_i!j+{Z24lau<^h_llU(fzw+w0jU~ zx>H|dQO9B4j-k=~I@K*~Z?0bI;xv{(s-e$ep8o)qb8V;Tp`UKKW>gQ5HME6nfyEa?)$QW)n}E3}nTe=OBUEX6GpKNMl@=}G(2x86~b z3y0K%FKsWDWFVAMzy&=1wV!wKLIz0|-qPiF{o<|^pI^_dbw@A7HOk_bozBIXP>-3v zs_L_9631t!vZOH~Yl-ALQZdt$->*vZ%bys z%{lv@GF%sk4)|Z~i z9PP!s$=sPN(LQP0@b#+8DJ_;Hlibqeliicg6|y=#Ckr~tskS)2ZKJC<@Q=jev~*iD z90=~abIPe4pZ1U8&{Rrrr?#xx6sJ?-(Hq(nK1(ToyiQ5Se)X%RTW>(hcps&6sW$F* z@~EpLuCtdg3>?-c+s&E8io+d9u87^3O32!pM^3#DdY~^n$i$J!xesw#A_iPX9GPYt zbKF-Y<3ANHmuPJ*oPD-+;~(w+sk~ll9GPlz8|MD|?*(07|Qz zmN70gaL00?106A1vNVN%)F}D`LToIKX|%Rh{{U9_`c~m?d5nIv?kgB)O2Y2mLG-P} zkIeo6K9pQ6bEX;M!1EuDm?pZI^wBPIa6Xx$=yqo_8qCCg@y>tRHPe4*#|h)8=yuY& z3o|-BKLagsF#iB_*0My3er&Mx=}ERUiZR%)d2R^*0It9NYSM~hYTKiX5<8V1{{Z7s z@6e@U6Q;)y0!6!0PCjW!{{Y!*pwhfR@M;lU+$7QjG7=-WWMjMx<2fC_8jpLaS{%IE zepLBWPCd+}zo4&L{{V?5xuo35V+GU9sD4(njf%E7%P7eD(@w&%<4vc!44b(};D5%w zk}VsD!GZV;)P~Igt?fHT#@~2)^(qUQOg;*(<6#F zV?xphu6*@ubCH}8`qQ{r=6)G#?WOp}7_V+*HurIxebPB0Sddin(~*zHmM;#7qY_C2 zZDEkigaSROIU(}1=nPkx>3%iUbjx(NxruKhL4{`}WXIRP;wfkrXQtw~ulP@;nlNC~ z0a{&a4I@mrzqvS$6f8$}J(PQs#d+qZcYUm~wd}#8Q@J8hgj4H} zUvo>DA*eIV<+H!No@<+lrr#^Hj5o0vB$NE>(O$}A3Jh|CC)@-do9HO#Wpbm=qS2<2 zxRyo1>PfFp(R?{2{-a^l@aC@<;Z2RXEP) zVl)2$*RCI^Nj+KE@#@+m!u8J#>pEO%eKo2{ZiYv>NS|vuXFIl@>-FiLmF#zVZN<&x zzIJ7duK|u!0zU2!UiH5$;VUI{VeKO&cFz>>&Y`2;>2GoieWU$rs1jTN;vGR&W74Rl4p^4<}qpFyM~Mv62}~bA6#Jn z04lff6#4!og@FZ{+yjqhHKjRoM2V|2m%fCFnpOLxD}Hr$?Fg_Wj*$RZ@UCZo2(2uM|gDOUc!opSNryeqLVS2-1&si_*mtfI6#4O?9M zL{?HF7kC-bk6fLEfA#B-mi9?gX*l_}V0|hfhlFosxpTx$R=+@mD}b#~BNa8HU5Sj> zQrz~EKsOP(Gtq@wv%JO?`BE{1Svg8O>U1h}=d-avYf<$0=J}u@82Hh%MquR%) z{e;%ql`XC%wvdwnQp@>Xf>oTwcu8Mbh;jzTesR2l#+33m>Od z{Y7O1RnEzDqRl&>4(c}Xd8-Jyx9u&vd5ijVs@AvHR#FKqZY72NV`4K+RUy#q$`V2j zMhGLdbfd)2a02R97I(wDx@>>@>1w^CNOWX!Wmyjfy2g{j*JJ07_Q8*)B~S6EbjjqB zIYB8MMr)@&AJVQr!&)V=#mt{D{t-r^(5>gWIL2yYbf_^DYNOSuTr9}^*#OHS0+FvA{pktz z6c6Son>87`_B5=@y6#LHHCyc`veL5-WLYMvnb`KH%EO%&395!hJ?Q0P=S7m$QzZ2? z`9;oIG7nmX z9>TY5V2&aHU;e#iD#>hZ3_E+W)`Oyc`ED*0I}TNQXZ&lR)0k}lcORem{VR^9B2PoQ zmKqV%yOnLKPs(~Q9VyB%<%c=@%H96H^_*HYac1_HarSZ$=RUr_<5<({Hd*} z!iVpJ>5A4Vy@-{T?CRt(T;P8fVd_1y}q7W)&s$O{_(#b;ruJ2H0EVXaV~^CVq=il+=Qtd4ewFFky^YqRrkiUxZS7$U7)>kThC7T8m+DVSxYX`Lis*Q& z$0{&pJRa51>RKhurNc)sU8>j;DayCgy=k0#agPn z6UxjjcR*y&VfBPF~z_ZoUzTxxemo zu1BUuK>TZHPN%OGT+n?*?;N&=;rUwLV?6T2JkqMWMou`%$gZ?7BZ4WYp*b^`)O~YTH&lRyvB!zZGYi%8! ztakAXM;O4(NY1K2W>qJqPAQ05W<&3t6n%d&WwUdY#)gS5ADl4O!bF2P50p5DWXS7S*Y;jeh|!p=|h zOP9Woa;py!^XmBY^aR(=-`aN?mG76Otg$f|;qWo)M?p~OCi$6aT@mf?V8TG)w%p1?PLILPnRSC!gY$!M(e-aOMc-Z}YxgFnut#ytvGIYFwOKTy>cYgtFy zTXGh{NehSGkJG63^);of&X@Lbq!#UQv!68w3Vyx*g09L|xRY_cj?Y5)ipk-Hh8rb& z=Nn~}$H)jheMvr?3i9hsQ|xj`u*L*0f)$V5IO|K;#`j2m)wO1I_u7@lp?UU;$-L>Y zh24M@dMF$UsF(o9LGwsMJd_12|v2s>yGR)s%MOH|aYrb* zjXg@^?K+L44ZsMqwT^c3c>OCVPr6X*q+vnL6_cZ5p<*KUHz(1eZ<+@y?0FRo zH`}9Ap4dFrAH9pU&Fw}?_A1L^a|q@#!_|4MRJ|og&ouq4*63cs4QS4PI5;C8THcYN zxGTOjIUMm;t9M3jgXqi=^{r_%+jGh@C+k$JMUd!C80A6!AIh{}N|{Dc)rY?oO00X3 z=*zL0mqYSM%T}Jo-VE!^%ZJJ&B8?A z+Zva9tB?Em-{Vz`WOO7`x+TU$uc_V0F!vKT)R`f9?C=+6<|QW-4t`6_A)EvRiCqu?ut3k{fvsa6>vbm-5-@6(GS>2r<%Lu zf!I>^h&_agT>2Wj<%&I`4`9lB81}285_$vqQSArp3VvDdSH8~u*B?`gUeJETrXu#M zqBUQ<>-cj^oNAIGjuTcy@W=cN=iJ~_xk>C=gjVC?gz<`vwMmp>B#u9Dxj&hu%QilE zMAo;qaxz92xjYZfx!JB}oaR|w{{Y@OrR*Esht^Wr9cb1pH1+xJ49-7-9)B~^xV+O? zSal`iC(!8cykC1i%WA7^xW?i0IUic)^IArXBeF4-dXn9eOEi;6WR=<7y0aXUQC5~R zy~WK*6fGkGieYBQTfoyu02R?`Rslvx=B3EiG@CDIwu=i8!=0xemPJ@Duv|jm;A8{- ze=42LQc*p5{4IM7{!oOzP6IDP{uSso$L$z$ql3Eewav2c+9g1_W>4X+^iZlX*&bkY zz1)Fbk@gEiCi8C|?B}IFXe%5AhJ_`>>k{x!b6$<*pnc&Nb}^iDiaFR=%F=W=^!?1w zy_t6L#w%P@xi&i$Zgksw$zwMvWr{S~u(A0w{o*^Eam`$!?3*0^rQtm?TX`k7fWbPc zW%9Zb2TU5>mNl9uiSx14t6<}UO-Z1!$7uc;p6bR6siqMLKw%c%LZ4GzmAnz!+s6b$ z5yqiZ6Py85=I@{+QM1w4PLw^R)XIeX@i`2PKcV-nJk;HVg&59w$?9n-1&8LDngl;I z+|UEFP0cVLnr#;W*{*BHdW`y4gEb4gY2F!>BBPR@GqCX3 z^6n&gZi??%5b_7B6T*+Je8I2lD|Na!}H&%Sv-jdVtv>dc&x={oO^FT76S zWM_`aSQz7rAqnh#T>JF&t{YF)lG4sOFJ4J5n}Ey*$~}PoeQC7zMJpW~-fQMdkG^>Z zN8G^s*E==;0NQ?Km^z-|^r^EeLt|IfwOvC*Z6@zgl3NoZ%eDS}rE`oPpmWoubB{TM zZZCoLt4*R>MH|}(hSqsi(p4$|F|arp=CiJ(^X`?EM%?}G=ianlW^E&&&@JukZ5nwb zcAiDt(T3Up9RdEmt13Hy$U}Y1<%h4J`cz3=xU`M9XE#NzXL2LX79|5~52@sj%Cc7I z&a85F^d6MCN=%qU7;BA zQ#n6V>GY{3K3WnFC#m(SA(_WL4{Dw`Ugg>Ig&%p3YGSbkK1N-|V+1F^Q$Py_1nOI$ zIi@MvaypKb0eM2_++3HO(4Xc)RrdfbJ zI+N)|#*=#tg3;!67$6>}917fZ1OPFPiaDvfFObiBXro-zWi%CGmqJShq%gT}{Q&P<%rp1K1W&6vYr9#7u)bhoebIo-wCD1s*{c45* z?g!&Z(?u%sHj?0#>-yG0?jLqQKj2lHqK#vTeNoUwa@%p5#aX9c;zR!c0u`H}YOy`d z_~$smBZ|crvIFxW&!{yngDGNSbO|8af@=+RB^^TwmqJA>L=Dm)WgDYYpZ04MYEk3w z9-g$ljK}S%KE|_J*c@gB=hGsxPNKb6ADuNyr*zRhF(wU2++tJzfUFKbJ&t>jo|h@1ZVHI8oF_oLZQxV^1D z#<`Bbbcp`|-)6DR(|XhPQtlqs9gR{f#&HNgRyB}kKD7Ol?k{Z~#+yrSy&+82SDXIT zKV(m~iJ;dYebhnp6`E!qqw}g%iFJ{6t{ndW7v)*zZ+fLjhg}vLPkhxwA@r(HGj)+U zlB?y9dVHk1q>S|z53N;^4sD8Bn~G#zhG@kx2~wm*yHdIfGB61|R+Y7??i;FHi&Wbn zx3gzCBDZ3SG~iS@wrZ&*xnA1TBQFB3Z~*o+(Cm?`Y@s6updz7JF%t9bO2(<^Lwehy UA$o&UuO;F|JdaAzBPmG#*`(OqY5)KL literal 0 HcmV?d00001 diff --git a/modules/control/units/xs_model.py b/modules/control/units/xs_model.py index fcb6d99b4..460ced12e 100644 --- a/modules/control/units/xs_model.py +++ b/modules/control/units/xs_model.py @@ -36,7 +36,10 @@ try: except Exception: pass -from diffusers.models.unet_2d_condition import UNet2DConditionModel +try: + from diffusers.models.unet_2d_condition import UNet2DConditionModel +except Exception: + from diffusers.models.unets.unet_2d_condition import UNet2DConditionModel from diffusers.utils import BaseOutput, logging, USE_PEFT_BACKEND diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 5b2361431..66f8b1b3f 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -100,6 +100,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): denoising_start=0 if use_refiner_start else p.refiner_start if use_denoise_start else None, denoising_end=p.refiner_start if use_refiner_start else 1 if use_denoise_start else None, output_type='latent' if hasattr(shared.sd_model, 'vae') else 'np', + # output_type='pil', clip_skip=p.clip_skip, desc='Base', ) @@ -115,7 +116,10 @@ def process_diffusers(p: processing.StableDiffusionProcessing): hidiffusion.apply_hidiffusion(p, shared.sd_model_type) # if 'image' in base_args: # base_args['image'] = set_latents(p) - output = shared.sd_model(**base_args) # pylint: disable=not-callable + if hasattr(shared.sd_model, 'tgate'): + output = shared.sd_model.tgate(**base_args) # pylint: disable=not-callable + else: + output = shared.sd_model(**base_args) if isinstance(output, dict): output = SimpleNamespace(**output) hidiffusion.remove_hidiffusion(p) @@ -305,7 +309,9 @@ def process_diffusers(p: processing.StableDiffusionProcessing): if not hasattr(output, 'images') and hasattr(output, 'frames'): shared.log.debug(f'Generated: frames={len(output.frames[0])}') output.images = output.frames[0] - if hasattr(shared.sd_model, "vae") and output.images is not None and len(output.images) > 0: + if torch.is_tensor(output.images) and len(output.images) > 0 and any(s >= 512 for s in output.images.shape): + results = output.images.cpu().numpy() + elif hasattr(shared.sd_model, "vae") and output.images is not None and len(output.images) > 0: results = processing_vae.vae_decode(latents=output.images, model=shared.sd_model, full_quality=p.full_quality) elif hasattr(output, 'images'): results = output.images diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index bf86f1e24..b093e71fd 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -503,12 +503,19 @@ def set_latents(p): return latents +last_circular = False def apply_circular(enable, model): + global last_circular # pylint: disable=global-statement + if not hasattr(model, 'unet') or not hasattr(model, 'vae'): + return + if last_circular == enable: + return try: for layer in [layer for layer in model.unet.modules() if type(layer) is torch.nn.Conv2d]: layer.padding_mode = 'circular' if enable else 'zeros' for layer in [layer for layer in model.vae.modules() if type(layer) is torch.nn.Conv2d]: layer.padding_mode = 'circular' if enable else 'zeros' + last_circular = enable except Exception as e: debug(f"Diffusers tiling failed: {e}") diff --git a/modules/sd_models.py b/modules/sd_models.py index 607606a94..826c8d708 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -604,6 +604,10 @@ def detect_pipeline(f: str, op: str = 'model', warning=True): if shared.backend == shared.Backend.ORIGINAL: warn(f'Model detected as SegMoE model, but attempting to load using backend=original: {op}={f} size={size} MB') guess = 'SegMoE' + if 'hunyuandit' in f.lower(): + if shared.backend == shared.Backend.ORIGINAL: + warn(f'Model detected as Tenecent HunyuanDiT model, but attempting to load using backend=original: {op}={f} size={size} MB') + guess = 'HunyuanDiT' if 'pixart-xl' in f.lower(): if shared.backend == shared.Backend.ORIGINAL: warn(f'Model detected as PixArt Alpha model, but attempting to load using backend=original: {op}={f} size={size} MB') @@ -1197,7 +1201,7 @@ def switch_pipe(cls: diffusers.DiffusionPipeline, pipeline: diffusers.DiffusionP new_pipe = None signature = inspect.signature(cls.__init__, follow_wrapped=True, eval_str=True) possible = signature.parameters.keys() - if isinstance(pipeline, cls): + if isinstance(pipeline, cls) and args == {}: return pipeline pipe_dict = {} components_used = [] diff --git a/modules/shared_items.py b/modules/shared_items.py index 25ed4ae78..fbcc26a69 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -90,6 +90,8 @@ def get_pipelines(): pipelines['Stable Cascade'] = getattr(diffusers, 'StableCascadeCombinedPipeline', None) if hasattr(diffusers, 'PixArtSigmaPipeline'): pipelines['PixArt-Sigma'] = getattr(diffusers, 'PixArtSigmaPipeline', None) + if hasattr(diffusers, 'HunyuanDiTPipeline'): + pipelines['HunyuanDiT'] = getattr(diffusers, 'HunyuanDiTPipeline', None) for k, v in pipelines.items(): if k != 'Autodetect' and v is None: diff --git a/scripts/kohya_hires_fix.py b/scripts/kohya_hires_fix.py new file mode 100644 index 000000000..2a50968af --- /dev/null +++ b/scripts/kohya_hires_fix.py @@ -0,0 +1,40 @@ +import gradio as gr +import diffusers +from modules import scripts, processing, shared, sd_models, devices + + +class Script(scripts.Script): + def title(self): + return 'Kohya HiRes Fix' + + def show(self, is_img2img): + return not is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + + # return signature is array of gradio components + def ui(self, _is_img2img): + with gr.Row(): + gr.HTML('  Kohya HiRes Fix
') + with gr.Row(): + enabled = gr.Checkbox(label="Enabled", value=True) + with gr.Row(): + scale_factor = gr.Slider(value=0.5, minimum=0, maximum=1, step=0.05, label="Scale factor") + timestep = gr.Number(value=600, minimum=0, maximum=1000, label="Timestep") + block_num = gr.Number(value=1, minimum=0, maximum=10, label="Block") + return [enabled, scale_factor, timestep, block_num] + + def run(self, p: processing.StableDiffusionProcessing, enabled, scale_factor, timestep, block_num): # pylint: disable=arguments-differ + if not enabled: + return None + if shared.sd_model_type != 'sd': + shared.log.warning(f'Kohya Hires Fix: pipeline={shared.sd_model_type} required=sd') + return None + old_pipe = shared.sd_model + high_res_fix = [{'timestep': timestep, 'scale_factor': scale_factor, 'block_num': block_num}] + shared.sd_model = diffusers.StableDiffusionPipeline.from_pipe(shared.sd_model, **{ 'custom_pipeline': 'kohya_hires_fix', 'high_res_fix': high_res_fix }) + sd_models.copy_diffuser_options(shared.sd_model, old_pipe) + sd_models.move_model(shared.sd_model, devices.device) # move pipeline to device + sd_models.set_diffuser_options(shared.sd_model, vae=None, op='model') + shared.log.debug(f'Kohya Hires Fix: pipeline={shared.sd_model.__class__.__name__} args={high_res_fix}') + processed = processing.process_images(p) + shared.sd_model = old_pipe + return processed diff --git a/scripts/t_gate.py b/scripts/t_gate.py new file mode 100644 index 000000000..72a1b5375 --- /dev/null +++ b/scripts/t_gate.py @@ -0,0 +1,43 @@ +import gradio as gr +from modules import scripts, processing, shared, sd_models, devices +from installer import install + + +class Script(scripts.Script): + def title(self): + return 'T-Gate' + + def show(self, is_img2img): + return not is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + + # return signature is array of gradio components + def ui(self, _is_img2img): + with gr.Row(): + gr.HTML('  T-Gate
') + with gr.Row(): + enabled = gr.Checkbox(label="Enabled", value=True) + with gr.Row(): + gate_step = gr.Slider(minimum=1, maximum=50, step=1, label="Gate step", elem_id="t_gate_steps", value=10) + return [enabled, gate_step] + + def run(self, p: processing.StableDiffusionProcessing, enabled, gate_step): # pylint: disable=arguments-differ + if not enabled: + return None + install('tgate') + import tgate + if shared.sd_model_type == 'sd': + cls = tgate.TgateSDLoader + elif shared.sd_model_type == 'sdxl': + cls = tgate.TgateSDXLLoader + else: + shared.log.warning(f'T-Gate: pipeline={shared.sd_model_type} required=sd or sdxl') + return None + old_pipe = shared.sd_model + shared.sd_model = cls(shared.sd_model, gate_step=min(gate_step, p.steps)) + sd_models.copy_diffuser_options(shared.sd_model, old_pipe) + sd_models.move_model(shared.sd_model, devices.device) # move pipeline to device + sd_models.set_diffuser_options(shared.sd_model, vae=None, op='model') + shared.log.debug(f'T-Gate: pipeline={shared.sd_model.__class__.__name__} steps={gate_step}') + processed = processing.process_images(p) + shared.sd_model = old_pipe + return processed From 901784f3b5a8930a01f2598505afb34fc70b8e64 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 3 Jun 2024 16:55:17 -0400 Subject: [PATCH 04/34] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b462c25ea..6609b35df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ *Note*: New features require `diffusers==0.29.0.dev` -- [Tenecent HunyuanDiT](https://github.com/Tencent/HunyuanDiT) bilingual english/chinese diffusion transformer model +- [Tenecent HunyuanDiT](https://github.com/Tencent/HunyuanDiT) bilingual english/chinese diffusion transformer model note: this is a very large model at ~17GB, but can be used with less VRAM using model offloading simply select from networks -> models -> reference, model will be auto-downloaded on first use - [MuLan](https://github.com/mulanai/MuLan) Multi-langunage prompts From 6a212a15f2cd85c41e686cee4e2298a3d08aac59 Mon Sep 17 00:00:00 2001 From: Luca Beltrame Date: Mon, 3 Jun 2024 23:51:18 +0200 Subject: [PATCH 05/34] Add two more ControlNets (scribble, one generic and one for anime) --- modules/control/units/controlnet.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index b7bf683fd..08840c304 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -51,7 +51,9 @@ predefined_sdxl = { 'Depth Mid XL': 'diffusers/controlnet-depth-sdxl-1.0-mid', 'OpenPose XL': 'thibaud/controlnet-openpose-sdxl-1.0', 'Xinsir OpenPose XL': 'xinsir/controlnet-openpose-sdxl-1.0', - 'Xinsir Canny': 'xinsir/controlnet-canny-sdxl-1.0' + 'Xinsir Canny': 'xinsir/controlnet-canny-sdxl-1.0', + 'Xinsir Scribble': 'xinsir/controlnet-scribble-sdxl-1.0', + 'Xinsir Anime Painter': 'xinsir/anime-painter', # 'StabilityAI Canny R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-canny-rank128.safetensors', # 'StabilityAI Depth R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-depth-rank128.safetensors', # 'StabilityAI Recolor R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-recolor-rank128.safetensors', From f84705d07961f207d5edd13487a62dd1276493e9 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 Jun 2024 07:12:12 -0400 Subject: [PATCH 06/34] fix tgate apply/unapply --- CHANGELOG.md | 8 ++++---- modules/processing_diffusers.py | 3 ++- scripts/t_gate.py | 6 ++++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6609b35df..3cef36f05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,9 +16,9 @@ write your prompts forin ~110 auto-detected languages! compatible with SD15 and SDXL enable in scripts -> MuLan and set encoder to `InternVL-14B-224px` encoder - *Note*: right now this is more of a proof-of-concept before smaller and/or quantized models are released + *note*: right now this is more of a proof-of-concept before smaller and/or quantized models are released model will be auto-downloaded on first use: note its huge size of 27GB - even executing it in FP16 context will require ~16GB of VRAM for text encoder alone + even executing it in FP16 will require ~16GB of VRAM for text encoder alone examples: - English: photo of a beautiful woman wearing a white bikini on a beach with a city skyline in the background - Croatian: fotografija lijepe žene u bijelom bikiniju na plaži s gradskim obzorom u pozadini @@ -35,8 +35,8 @@ download from - **Kohya HiRes Fix** allows for higher resolution generation using standard sd15 models enable via scripts -> kohya-hires-fix - *note*: this alternative to regular hidiffusion method, but with different approach to scaling -- additional built-in controlnet models: TODO + *note*: alternative to regular hidiffusion method, but with different approach to scaling +- additional built-in **ControlNet** models: TODO - lower overhead on generate calls - cumulative fixes since the last release diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 66f8b1b3f..8b8326aca 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -116,7 +116,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing): hidiffusion.apply_hidiffusion(p, shared.sd_model_type) # if 'image' in base_args: # base_args['image'] = set_latents(p) - if hasattr(shared.sd_model, 'tgate'): + if hasattr(shared.sd_model, 'tgate') and getattr(p, 'gate_step', -1) > 0: + base_args['gate_step'] = p.gate_step output = shared.sd_model.tgate(**base_args) # pylint: disable=not-callable else: output = shared.sd_model(**base_args) diff --git a/scripts/t_gate.py b/scripts/t_gate.py index 72a1b5375..7c55f998d 100644 --- a/scripts/t_gate.py +++ b/scripts/t_gate.py @@ -21,6 +21,7 @@ class Script(scripts.Script): return [enabled, gate_step] def run(self, p: processing.StableDiffusionProcessing, enabled, gate_step): # pylint: disable=arguments-differ + p.gate_step = min(gate_step, p.steps) if enabled else -1 if not enabled: return None install('tgate') @@ -33,11 +34,12 @@ class Script(scripts.Script): shared.log.warning(f'T-Gate: pipeline={shared.sd_model_type} required=sd or sdxl') return None old_pipe = shared.sd_model - shared.sd_model = cls(shared.sd_model, gate_step=min(gate_step, p.steps)) + shared.sd_model = cls(shared.sd_model, gate_step=p.gate_step) sd_models.copy_diffuser_options(shared.sd_model, old_pipe) sd_models.move_model(shared.sd_model, devices.device) # move pipeline to device sd_models.set_diffuser_options(shared.sd_model, vae=None, op='model') - shared.log.debug(f'T-Gate: pipeline={shared.sd_model.__class__.__name__} steps={gate_step}') + shared.log.debug(f'T-Gate: pipeline={shared.sd_model.__class__.__name__} steps={p.gate_step}') processed = processing.process_images(p) shared.sd_model = old_pipe + del shared.sd_model.tgate return processed From e15c21ede9c60dce505d15bae90ae436851703f0 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 Jun 2024 07:56:46 -0400 Subject: [PATCH 07/34] facehires implement include mask in output --- modules/masking.py | 2 +- modules/processing.py | 8 ++++++-- modules/shared.py | 4 ++-- scripts/face-details.py | 10 ++++++---- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/modules/masking.py b/modules/masking.py index 729c8fc63..dc071eead 100644 --- a/modules/masking.py +++ b/modules/masking.py @@ -442,7 +442,7 @@ def run_mask(input_image: Image.Image, input_mask: Image.Image = None, return_ty return_type = return_type or opts.preview_type - shared.log.debug(f'Mask: size={input_image.width}x{input_image.height} masked={mask_size}px area={area_size/total_size:.2f} auto={opts.auto_mask} blur={opts.mask_blur} erode={opts.mask_erode} dilate={opts.mask_dilate} type={return_type} time={t1-t0:.2f}') + shared.log.debug(f'Mask: size={input_image.width}x{input_image.height} masked={mask_size}px area={area_size/total_size:.2f} auto={opts.auto_mask} blur={opts.mask_blur:.3f} erode={opts.mask_erode:.3f} dilate={opts.mask_dilate:.3f} type={return_type} time={t1-t0:.2f}') if return_type == 'None': return input_mask elif return_type == 'Opaque': diff --git a/modules/processing.py b/modules/processing.py index 211f790b8..e0570d42a 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -3,7 +3,7 @@ import json import time from contextlib import nullcontext import numpy as np -from PIL import Image +from PIL import Image, ImageOps from modules import shared, devices, errors, images, scripts, memstats, lowvram, script_callbacks, extra_networks, face_restoration, sd_hijack_freeu, sd_models, sd_vae, processing_helpers from modules.sd_hijack_hypertile import context_hypertile_vae, context_hypertile_unet from modules.processing_class import StableDiffusionProcessing, StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, StableDiffusionProcessingControl # pylint: disable=unused-import @@ -415,7 +415,11 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: extra_networks.deactivate(p, extra_network_data) if shared.opts.include_mask: - if getattr(p, 'image_mask', None) is not None and isinstance(p.image_mask, Image.Image): + if shared.opts.mask_apply_overlay and p.overlay_images is not None and len(p.overlay_images): + p.image_mask = create_binary_mask(p.overlay_images[0]) + p.image_mask = ImageOps.invert(p.image_mask) + output_images.append(p.image_mask) + elif getattr(p, 'image_mask', None) is not None and isinstance(p.image_mask, Image.Image): if getattr(p, 'mask_for_facehires', None) is not None: output_images.append(p.mask_for_facehires) else: diff --git a/modules/shared.py b/modules/shared.py index d71efd5df..2de3dc505 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -752,10 +752,10 @@ options_templates.update(options_section(('postprocessing', "Postprocessing"), { "facehires_iou": OptionInfo(0.5, "Max face overlap", gr.Slider, {"minimum": 0, "maximum": 1.0, "step": 0.05}), "facehires_min_size": OptionInfo(0, "Min face size", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 1}), "facehires_max_size": OptionInfo(0, "Max face size", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 1}), - "facehires_padding": OptionInfo(10, "Face padding", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}), - "face_restoration_unload": OptionInfo(False, "Move model to CPU when complete"), + "facehires_padding": OptionInfo(20, "Face padding", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}), "facehires_strength": OptionInfo(0.0, "Face restore strength", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}), "code_former_weight": OptionInfo(0.2, "CodeFormer weight parameter", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}), + "face_restoration_unload": OptionInfo(False, "Move model to CPU when complete"), "postprocessing_sep_upscalers": OptionInfo("

Upscaling

", "", gr.HTML), "upscaler_unload": OptionInfo(False, "Unload upscaler after processing"), diff --git a/scripts/face-details.py b/scripts/face-details.py index 849461c77..f5e554f15 100644 --- a/scripts/face-details.py +++ b/scripts/face-details.py @@ -160,6 +160,7 @@ class FaceRestorerYolo(FaceRestoration): continue p.init_images = [image] p.image_mask = [face.mask] + # mask_all.append(face.mask) p.recursion = True pp = processing.process_images_inner(p) del p.recursion @@ -176,12 +177,13 @@ class FaceRestorerYolo(FaceRestoration): shared.opts.data['mask_apply_overlay'] = orig_apply_overlay np_image = np.array(image) - """ if len(mask_all) > 0 and shared.opts.include_mask: from modules.control.util import blend - mask_all = blend([np.array(m) for m in mask_all]) - mask_pil = Image.fromarray(mask_all) - """ + p.image_mask = blend([np.array(m) for m in mask_all]) + # combined = blend([np_image, p.image_mask]) + # combined = Image.fromarray(combined) + # combined.save('/tmp/face.png') + p.image_mask = Image.fromarray(p.image_mask) return np_image From 4336e3abf408bc20a7487dc9261021e52960c2e9 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 Jun 2024 08:07:54 -0400 Subject: [PATCH 08/34] update dit path --- CHANGELOG.md | 13 +++++++------ html/reference.json | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cef36f05..fb2bcc39e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,6 @@ ## TODO -- update checkpoint: `Tencent-Hunyuan/HunyuanDiT-Diffusers` - add controlnets ## Update for 2024-06-03 @@ -14,7 +13,7 @@ simply select from networks -> models -> reference, model will be auto-downloaded on first use - [MuLan](https://github.com/mulanai/MuLan) Multi-langunage prompts write your prompts forin ~110 auto-detected languages! - compatible with SD15 and SDXL + compatible with *SD15* and *SDXL* enable in scripts -> MuLan and set encoder to `InternVL-14B-224px` encoder *note*: right now this is more of a proof-of-concept before smaller and/or quantized models are released model will be auto-downloaded on first use: note its huge size of 27GB @@ -30,13 +29,15 @@ - Chinese: 一个美丽的女人在海滩上穿着白色比基尼的照片, 背景是城市天际线 - Korean: 도시의 스카이라인을 배경으로 해변에서 흰색 비키니를 입은 아름 다운 여성의 사진 - [T-Gate](https://github.com/HaozheLiu-ST/T-GATE) Speed up generations by gating at which step cross-attention is no longer needed - enable via scripts -> t-gate -- **PCM LoRAs** allow for fast denoising using less steps with standard sd15 and sdxl models + enable via scripts -> t-gate + compatible with *SD15* +- **PCM LoRAs** allow for fast denoising using less steps with standard *SD15* and *SDXL* models download from -- **Kohya HiRes Fix** allows for higher resolution generation using standard sd15 models +- **Kohya HiRes Fix** allows for higher resolution generation using standard *SD15* models enable via scripts -> kohya-hires-fix *note*: alternative to regular hidiffusion method, but with different approach to scaling -- additional built-in **ControlNet** models: TODO +- additional built-in 4 great **ControlNet** SDXL models from Xinsir: OpenPose, Canny, Scribble, AnimePainter + thanks @lbeltrame - lower overhead on generate calls - cumulative fixes since the last release diff --git a/html/reference.json b/html/reference.json index 458612bf6..aedafee3d 100644 --- a/html/reference.json +++ b/html/reference.json @@ -159,7 +159,7 @@ }, "Tencent HunyuanDiT": { - "path": "XCLiu/HunyuanDiT-0523", + "path": "Tencent-Hunyuan/HunyuanDiT-Diffusers", "desc": "Hunyuan-DiT : A Powerful Multi-Resolution Diffusion Transformer with Fine-Grained Chinese Understanding.", "preview": "Tencent-Hunyuan-HunyuanDiT.jpg", "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 2.0" From 28b240f51b291c589a250aff885a64e7a5a332ef Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 Jun 2024 08:08:37 -0400 Subject: [PATCH 09/34] update --- modules/control/units/controlnet.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index 08840c304..e911bc6ff 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -51,9 +51,9 @@ predefined_sdxl = { 'Depth Mid XL': 'diffusers/controlnet-depth-sdxl-1.0-mid', 'OpenPose XL': 'thibaud/controlnet-openpose-sdxl-1.0', 'Xinsir OpenPose XL': 'xinsir/controlnet-openpose-sdxl-1.0', - 'Xinsir Canny': 'xinsir/controlnet-canny-sdxl-1.0', - 'Xinsir Scribble': 'xinsir/controlnet-scribble-sdxl-1.0', - 'Xinsir Anime Painter': 'xinsir/anime-painter', + 'Xinsir Canny XL': 'xinsir/controlnet-canny-sdxl-1.0', + 'Xinsir Scribble XL': 'xinsir/controlnet-scribble-sdxl-1.0', + 'Xinsir Anime Painter XL': 'xinsir/anime-painter', # 'StabilityAI Canny R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-canny-rank128.safetensors', # 'StabilityAI Depth R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-depth-rank128.safetensors', # 'StabilityAI Recolor R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-recolor-rank128.safetensors', From aba60270900cc638b5e4cbe9898b95ecbb61b29d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 Jun 2024 12:11:25 -0400 Subject: [PATCH 10/34] update changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb2bcc39e..08f3da97d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## TODO -- add controlnets +- StableDiffusion 3 ## Update for 2024-06-03 @@ -36,7 +36,7 @@ - **Kohya HiRes Fix** allows for higher resolution generation using standard *SD15* models enable via scripts -> kohya-hires-fix *note*: alternative to regular hidiffusion method, but with different approach to scaling -- additional built-in 4 great **ControlNet** SDXL models from Xinsir: OpenPose, Canny, Scribble, AnimePainter +- additional built-in 4 great custom trained **ControlNet SDXL** models from Xinsir: OpenPose, Canny, Scribble, AnimePainter thanks @lbeltrame - lower overhead on generate calls - cumulative fixes since the last release From 900264a194f440fce4e1720888e17e3c78fbca1d Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Wed, 5 Jun 2024 10:38:03 +0900 Subject: [PATCH 11/34] windows conceal rocm if zluda disabled --- installer.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/installer.py b/installer.py index 88f50e5d4..9f23e4786 100644 --- a/installer.py +++ b/installer.py @@ -540,6 +540,16 @@ def check_torch(): log.info("For ZLUDA support specify '--use-zluda'") log.info('Using CPU-only torch') torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') + + # conceal ROCm installed + os.environ.pop("ROCM_HOME", None) + os.environ.pop("ROCM_PATH", None) + paths = os.environ["PATH"].split(";") + paths_no_rocm = [] + for path in paths: + if "ROCm" not in path: + paths_no_rocm.append(path) + os.environ["PATH"] = ";".join(paths_no_rocm) else: if rocm_ver is None: # assume the latest if version check fails torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/rocm6.0') From 45d1cddbce6b3db7a3ee87389e533c35cba45012 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 5 Jun 2024 17:24:22 -0400 Subject: [PATCH 12/34] add directml python version check --- CHANGELOG.md | 1 + extensions-builtin/sdnext-modernui | 2 +- installer.py | 6 ++++-- modules/layerdiffuse/layerdiffuse_model.py | 5 ++++- modules/xadapter/unet_adapter.py | 12 +++++------- requirements.txt | 15 +++++++-------- 6 files changed, 22 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08f3da97d..af4307967 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ thanks @lbeltrame - lower overhead on generate calls - cumulative fixes since the last release +- add python version check for torch-directml ## Update for 2024-06-02 diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 0b56557c1..8afbad75d 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 0b56557c15467d6c86b9ef1d6cbfd55bd2f52928 +Subproject commit 8afbad75d6cd238270111ec77ff19b567855d8bd diff --git a/installer.py b/installer.py index 9f23e4786..328c57b29 100644 --- a/installer.py +++ b/installer.py @@ -386,13 +386,14 @@ def get_platform(): # check python version -def check_python(): - supported_minors = [9, 10, 11] +def check_python(supported_minors=[9, 10, 11], reason=None): if args.quick: return log.info(f'Python {platform.python_version()} on {platform.system()}') if not (int(sys.version_info.major) == 3 and int(sys.version_info.minor) in supported_minors): log.error(f"Incompatible Python version: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro} required 3.{supported_minors}") + if reason is not None: + log.error(reason) if not args.ignore: sys.exit(1) if not args.skip_git: @@ -619,6 +620,7 @@ def check_torch(): torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') elif allow_directml and args.use_directml and ('arm' not in machine and 'aarch' not in machine): log.info('Using DirectML Backend') + check_python(supported_minors=[10], reason='DirectML backend requires Python 3.10') torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.0.0 torchvision torch-directml') if 'torch' in torch_command and not args.version: install(torch_command, 'torch torchvision') diff --git a/modules/layerdiffuse/layerdiffuse_model.py b/modules/layerdiffuse/layerdiffuse_model.py index a0eb71cd2..9a2259462 100644 --- a/modules/layerdiffuse/layerdiffuse_model.py +++ b/modules/layerdiffuse/layerdiffuse_model.py @@ -9,9 +9,12 @@ from typing import Optional, Tuple, Union from diffusers import AutoencoderKL from diffusers.configuration_utils import ConfigMixin, register_to_config from diffusers.models.modeling_utils import ModelMixin -from diffusers.models.unet_2d_blocks import UNetMidBlock2D, get_down_block, get_up_block from diffusers.models.autoencoders.vae import DecoderOutput from diffusers.models.attention_processor import Attention, AttnProcessor +try: + from diffusers.models.unet_2d_blocks import UNetMidBlock2D, get_down_block, get_up_block +except Exception: + from diffusers.models.unets.unet_2d_blocks import UNetMidBlock2D, get_down_block, get_up_block def zero_module(module): diff --git a/modules/xadapter/unet_adapter.py b/modules/xadapter/unet_adapter.py index fa11c7cf8..5022f1847 100644 --- a/modules/xadapter/unet_adapter.py +++ b/modules/xadapter/unet_adapter.py @@ -28,20 +28,18 @@ from diffusers.models.embeddings import ( ImageHintTimeEmbedding, ImageProjection, ImageTimeEmbedding, - PositionNet, TextImageProjection, TextImageTimeEmbedding, TextTimeEmbedding, TimestepEmbedding, Timesteps, ) +from modules.xadapter.xadapter_hijacks import PositionNet from diffusers.models.modeling_utils import ModelMixin -from diffusers.models.unet_2d_blocks import ( - UNetMidBlock2DCrossAttn, - UNetMidBlock2DSimpleCrossAttn, - get_down_block, - get_up_block, -) +try: + from diffusers.models.unet_2d_blocks import UNetMidBlock2DCrossAttn, UNetMidBlock2DSimpleCrossAttn, get_down_block, get_up_block +except Exception: + from diffusers.models.unets.unet_2d_blocks import UNetMidBlock2DCrossAttn, UNetMidBlock2DSimpleCrossAttn, get_down_block, get_up_block logger = logging.get_logger(__name__) # pylint: disable=invalid-name diff --git a/requirements.txt b/requirements.txt index 8aaf22fbc..17b7f6e16 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ patch-ng anyio addict astunparse -blendmodes clean-fid filetype future @@ -15,32 +14,27 @@ kornia lark lpips omegaconf -open-clip-torch optimum piexif psutil pyyaml resize-right rich -scipy toml -torchdiffeq voluptuous yapf -scikit-image fasteners -dctorch -pymatting orjson invisible-watermark pi-heif -diffusers==0.28.0 +diffusers==0.28.1 safetensors==0.4.3 tensordict==0.1.2 peft==0.11.1 httpx==0.24.1 compel==2.0.2 torchsde==0.2.6 +open-clip-torch clip-interrogator==0.6.0 antlr4-python3-runtime==4.9.3 requests==2.31.0 @@ -53,6 +47,8 @@ huggingface_hub==0.23.2 numexpr==2.8.8 numpy==1.26.4 numba==0.59.1 +blendmodes +scipy pandas protobuf==4.25.3 pytorch_lightning==1.9.4 @@ -63,3 +59,6 @@ Pillow==10.3.0 timm==0.9.16 pydantic==1.10.15 typing-extensions==4.11.0 +torchdiffeq +dctorch +scikit-image From 8a9c10e66582c465da155043afacd2f447e37fe4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 5 Jun 2024 17:36:43 -0400 Subject: [PATCH 13/34] hidiffusion disable embeds cache --- modules/hidiffusion/__init__.py | 2 +- modules/prompt_parser_diffusers.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/hidiffusion/__init__.py b/modules/hidiffusion/__init__.py index 50a52630a..e8b5f0fd0 100644 --- a/modules/hidiffusion/__init__.py +++ b/modules/hidiffusion/__init__.py @@ -10,7 +10,7 @@ def apply_hidiffusion(p, model_type): shared.log.warning(f'HiDiffusion: class={shared.sd_model.__class__.__name__} not supported') return remove_hidiffusion(p) - if p.hidiffusion: + if getattr(p, 'hidiffusion', False) is True: t0 = time.time() hidiffusion.is_aggressive_raunet = shared.opts.hidiffusion_steps > 0 hidiffusion.aggressive_step = shared.opts.hidiffusion_steps diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index bfe267e2e..7f6b90d6d 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -130,6 +130,8 @@ def get_tokens(msg, prompt): def encode_prompts(pipe, p, prompts: list, negative_prompts: list, steps: int, clip_skip: typing.Optional[int] = None): + if getattr(p, 'hidiffusion', False) is True: + cache.clear() if 'StableDiffusion' not in pipe.__class__.__name__ and 'DemoFusion' not in pipe.__class__.__name__ and 'StableCascade' not in pipe.__class__.__name__: shared.log.warning(f"Prompt parser not supported: {pipe.__class__.__name__}") return From 9bc217644d4665c56b54f0ede715fa328b86dcf1 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 6 Jun 2024 11:24:31 -0400 Subject: [PATCH 14/34] improve metadata parser --- CHANGELOG.md | 3 +- cli/image-exif.py | 65 +--------- modules/api/endpoints.py | 4 +- modules/generation_parameters_copypaste.py | 136 +-------------------- modules/infotext.py | 127 +++++++++++++++++++ modules/postprocessing.py | 20 +-- modules/prompt_parser_diffusers.py | 2 - modules/styles.py | 7 +- modules/ui_common.py | 10 +- modules/ui_extra_networks.py | 8 +- 10 files changed, 162 insertions(+), 220 deletions(-) create mode 100644 modules/infotext.py diff --git a/CHANGELOG.md b/CHANGELOG.md index af4307967..abbbc2441 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - StableDiffusion 3 -## Update for 2024-06-03 +## Update for 2024-06-04 *Note*: New features require `diffusers==0.29.0.dev` @@ -41,6 +41,7 @@ - lower overhead on generate calls - cumulative fixes since the last release - add python version check for torch-directml +- improve metadata/infotext parser ## Update for 2024-06-02 diff --git a/cli/image-exif.py b/cli/image-exif.py index f7268fde8..151a0df99 100755 --- a/cli/image-exif.py +++ b/cli/image-exif.py @@ -4,71 +4,16 @@ import os import io import re import sys -import json +import importlib from PIL import Image, ExifTags, TiffImagePlugin, PngImagePlugin from rich import print # pylint: disable=redefined-builtin -def unquote(text): - if len(text) == 0 or text[0] != '"' or text[-1] != '"': - return text - try: - return json.loads(text) - except Exception: - return text +module_spec = importlib.util.spec_from_file_location('infotext', os.path.join('modules', 'infotext.py')) +infotext = importlib.util.module_from_spec(module_spec) +module_spec.loader.exec_module(infotext) -def parse_generation_parameters(infotext): - if not isinstance(infotext, str): - return {} - re_param = re.compile(r'\s*([\w ]+):\s*("(?:\\"[^,]|\\"|\\|[^\"])+"|[^,]*)(?:,|$)') # multi-word: value - re_size = re.compile(r"^(\d+)x(\d+)$") # int x int - basic_params = ['steps', 'seed', 'width', 'height', 'sampler', 'size', 'cfg scale', 'hires'] # first param is one of those - - sanitized = infotext.replace('prompt:', 'Prompt:').replace('negative prompt:', 'Negative prompt:').replace('Negative Prompt', 'Negative prompt') # cleanup everything in brackets so re_params can work - sanitized = re.sub(r'<[^>]*>', lambda match: ' ' * len(match.group()), sanitized) - sanitized = re.sub(r'\([^)]*\)', lambda match: ' ' * len(match.group()), sanitized) - sanitized = re.sub(r'\{[^}]*\}', lambda match: ' ' * len(match.group()), sanitized) - - params = dict(re_param.findall(sanitized)) - params = { k.strip():params[k].strip() for k in params if k.lower() not in ['hashes', 'lora', 'embeddings', 'prompt', 'negative prompt']} # remove some keys - if len(list(params)) == 0: - first_param = None - else: - try: - first_param, first_param_idx = next((s, i) for i, s in enumerate(params) if any(x in s.lower() for x in basic_params)) - except Exception: - first_param, first_param_idx = next(iter(params)), 0 - if first_param_idx > 0: - for _i in range(first_param_idx): - params.pop(next(iter(params))) - params_idx = sanitized.find(f'{first_param}:') if first_param else -1 - negative_idx = infotext.find("Negative prompt:") - - prompt = infotext[:params_idx] if negative_idx == -1 else infotext[:negative_idx] # prompt can be with or without negative prompt - negative = infotext[negative_idx:params_idx] if negative_idx >= 0 else '' - - for k, v in params.copy().items(): # avoid dict-has-changed - if len(v) > 0 and v[0] == '"' and v[-1] == '"': - v = unquote(v) - m = re_size.match(v) - if v.replace('.', '', 1).isdigit(): - params[k] = float(v) if '.' in v else int(v) - elif v == "True": - params[k] = True - elif v == "False": - params[k] = False - elif m is not None: - params[f"{k}-1"] = int(m.group(1)) - params[f"{k}-2"] = int(m.group(2)) - elif k == 'VAE' and v == 'TAESD': - params["Full quality"] = False - else: - params[k] = v - params["Prompt"] = prompt.replace('Prompt:', '').strip() - params["Negative prompt"] = negative.replace('Negative prompt:', '').strip() - return params - class Exif: # pylint: disable=single-string-used-for-slots __slots__ = ('__dict__') # pylint: disable=superfluous-parens @@ -132,7 +77,7 @@ class Exif: # pylint: disable=single-string-used-for-slots def parse(self): x = self.exif.pop('parameters', None) or self.exif.pop('UserComment', None) - res = parse_generation_parameters(x) + res = infotext.parse(x) return res def get_bytes(self): diff --git a/modules/api/endpoints.py b/modules/api/endpoints.py index 6081c85d0..63c5764c0 100644 --- a/modules/api/endpoints.py +++ b/modules/api/endpoints.py @@ -146,7 +146,7 @@ def get_extensions_list(): return ext_list def post_pnginfo(req: models.ReqImageInfo): - from modules import images, script_callbacks, generation_parameters_copypaste + from modules import images, script_callbacks, infotext if not req.image.strip(): return models.ResImageInfo(info="") image = helpers.decode_base64_to_image(req.image.strip()) @@ -155,6 +155,6 @@ def post_pnginfo(req: models.ReqImageInfo): geninfo, items = images.read_info_from_image(image) if geninfo is None: geninfo = "" - params = generation_parameters_copypaste.parse_generation_parameters(geninfo) + params = infotext.parse(geninfo) script_callbacks.infotext_pasted_callback(geninfo, params) return models.ResImageInfo(info=geninfo, items=items, parameters=params) diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py index 97cf6d38f..feebfe50e 100644 --- a/modules/generation_parameters_copypaste.py +++ b/modules/generation_parameters_copypaste.py @@ -1,12 +1,11 @@ import base64 import io import os -import re -import json from PIL import Image import gradio as gr from modules.paths import data_path from modules import shared, gr_tempdir, script_callbacks, images +from modules.infotext import parse, mapping, quote, unquote # pylint: disable=unused-import type_of_gr_update = type(gr.update()) @@ -14,7 +13,8 @@ paste_fields = {} registered_param_bindings = [] debug = shared.log.trace if os.environ.get('SD_PASTE_DEBUG', None) is not None else lambda *args, **kwargs: None debug('Trace: PASTE') - +parse_generation_parameters = parse # compatibility +infotext_to_setting_name_mapping = mapping # compatibility class ParamBinding: def __init__(self, paste_button, tabname, source_text_component=None, source_image_component=None, source_tabname=None, override_settings_component=None, paste_field_names=None): @@ -32,21 +32,6 @@ def reset(): paste_fields.clear() -def quote(text): - if ',' not in str(text) and '\n' not in str(text) and ':' not in str(text): - return text - return json.dumps(text, ensure_ascii=False) - - -def unquote(text): - if len(text) == 0 or text[0] != '"' or text[-1] != '"': - return text - try: - return json.loads(text) - except Exception: - return text - - def image_from_url_text(filedata): if filedata is None: return None @@ -187,124 +172,13 @@ def send_image_and_dimensions(x): return img, w, h -def parse_generation_parameters(infotext, no_prompt=False): - if not isinstance(infotext, str): - return {} - debug(f'Parse infotext: {infotext}') - re_param = re.compile(r'\s*([\w ]+):\s*("(?:\\"[^,]|\\"|\\|[^\"])+"|[^,]*)(?:,|$)') # multi-word: value - re_size = re.compile(r"^(\d+)x(\d+)$") # int x int - basic_params = ['steps:', 'seed:', 'width:', 'height:', 'sampler:', 'size:', 'cfg scale:'] # first param is one of those - - infotext = infotext.replace('prompt:', 'Prompt:').replace('negative prompt:', 'Negative prompt:').replace('Negative Prompt', 'Negative prompt') # cleanup everything in brackets so re_params can work - infotext = infotext.replace(' Steps: ', ', Steps: ').replace('\nSteps: ', ', Steps: ') # fix cases where there is no delimiter between prompt and steps - sanitized = infotext - sanitized = re.sub(r'<[^>]*>', lambda match: ' ' * len(match.group()), sanitized) - sanitized = re.sub(r'\([^)]*\)', lambda match: ' ' * len(match.group()), sanitized) - sanitized = re.sub(r'\{[^}]*\}', lambda match: ' ' * len(match.group()), sanitized) - - params = dict(re_param.findall(sanitized)) - debug(f"Parse params: {params}") - params = { k.strip():params[k].strip() for k in params if k.lower() not in ['hashes', 'lora', 'embeddings', 'prompt', 'negative prompt']} # remove some keys - if len(list(params)) == 0: - first_param = None - else: - try: - first_param, first_param_idx = next((s, i) for i, s in enumerate(params) if any(x in s.lower() for x in basic_params)) - except Exception: - first_param, first_param_idx = next(iter(params)), 0 - if first_param_idx > 0: - for _i in range(first_param_idx): - params.pop(next(iter(params))) - params_idx = sanitized.find(f'{first_param}:') if first_param else -1 - negative_idx = infotext.find("Negative prompt:") - if 'Steps:' in sanitized: - params_idx = max(params_idx, sanitized.find('Steps:')) - - if negative_idx == -1: # prompt can be without negative prompt - prompt = infotext[:params_idx] if params_idx > 0 else infotext - else: - prompt = infotext[:negative_idx] - if prompt.startswith('Steps: '): - prompt = '' - if negative_idx >= 0: - negative = infotext[negative_idx:params_idx] if params_idx > 0 else infotext[negative_idx:] - else: - negative = '' - - for k, v in params.copy().items(): # avoid dict-has-changed - if len(v) > 0 and v[0] == '"' and v[-1] == '"': - v = unquote(v) - m = re_size.match(v) - if v.replace('.', '', 1).isdigit(): - params[k] = float(v) if '.' in v else int(v) - elif v == "True": - params[k] = True - elif v == "False": - params[k] = False - elif m is not None: - params[f"{k}-1"] = int(m.group(1)) - params[f"{k}-2"] = int(m.group(2)) - elif k == 'VAE' and v == 'TAESD': - params["Full quality"] = False - else: - params[k] = v - if not no_prompt: - params["Prompt"] = prompt.replace('Prompt:', '').strip(' ,\n') - params["Negative prompt"] = negative.replace('Negative prompt:', '').strip(' ,\n') - debug(f"Parse: {params}") - return params - - -settings_map = {} - - -infotext_to_setting_name_mapping = [ - ('Backend', 'sd_backend'), - ('Model hash', 'sd_model_checkpoint'), - ('Refiner', 'sd_model_refiner'), - ('VAE', 'sd_vae'), - ('Parser', 'prompt_attention'), - ('Color correction', 'img2img_color_correction'), - # Samplers - ('Sampler Eta', 'scheduler_eta'), - ('Sampler ENSD', 'eta_noise_seed_delta'), - ('Sampler order', 'schedulers_solver_order'), - # Samplers diffusers - ('Sampler beta schedule', 'schedulers_beta_schedule'), - ('Sampler beta start', 'schedulers_beta_start'), - ('Sampler beta end', 'schedulers_beta_end'), - ('Sampler DPM solver', 'schedulers_dpm_solver'), - # Samplers original - ('Sampler brownian', 'schedulers_brownian_noise'), - ('Sampler discard', 'schedulers_discard_penultimate'), - ('Sampler dyn threshold', 'schedulers_use_thresholding'), - ('Sampler karras', 'schedulers_use_karras'), - ('Sampler low order', 'schedulers_use_loworder'), - ('Sampler quantization', 'enable_quantization'), - ('Sampler sigma', 'schedulers_sigma'), - ('Sampler sigma min', 's_min'), - ('Sampler sigma max', 's_max'), - ('Sampler sigma churn', 's_churn'), - ('Sampler sigma uncond', 's_min_uncond'), - ('Sampler sigma noise', 's_noise'), - ('Sampler sigma tmin', 's_tmin'), - ('Sampler ENSM', 'initial_noise_multiplier'), # img2img only - ('UniPC skip type', 'uni_pc_skip_type'), - ('UniPC variant', 'uni_pc_variant'), - # Token Merging - ('Mask weight', 'inpainting_mask_weight'), - ('ToMe', 'tome_ratio'), - ('ToDo', 'todo_ratio'), -] - - def create_override_settings_dict(text_pairs): res = {} params = {} for pair in text_pairs: k, v = pair.split(":", maxsplit=1) params[k] = v.strip() - for param_name, setting_name in infotext_to_setting_name_mapping: + for param_name, setting_name in mapping: value = params.get(param_name, None) if value is None: continue @@ -325,7 +199,7 @@ def connect_paste(button, local_paste_fields, input_comp, override_settings_comp prompt = '' else: shared.log.debug(f'Paste prompt: type="current" prompt="{prompt}"') - params = parse_generation_parameters(prompt, no_prompt=False) + params = parse(prompt) script_callbacks.infotext_pasted_callback(prompt, params) res = [] applied = {} diff --git a/modules/infotext.py b/modules/infotext.py new file mode 100644 index 000000000..ed569517d --- /dev/null +++ b/modules/infotext.py @@ -0,0 +1,127 @@ +import os +import re +import json + + +debug = lambda *args, **kwargs: None # pylint: disable=unnecessary-lambda-assignment +re_size = re.compile(r"^(\d+)x(\d+)$") # int x int +re_param = re.compile(r'\s*([\w ]+):\s*("(?:\\"[^,]|\\"|\\|[^\"])+"|[^,]*)(?:,|$)') # multi-word: value + + +def quote(text): + if ',' not in str(text) and '\n' not in str(text) and ':' not in str(text): + return text + return json.dumps(text, ensure_ascii=False) + + +def unquote(text): + if len(text) == 0 or text[0] != '"' or text[-1] != '"': + return text + try: + return json.loads(text) + except Exception: + return text + + +def parse(infotext): + if not isinstance(infotext, str): + return {} + debug(f'Raw: {infotext}') + if 'negative prompt:' not in infotext: + infotext = 'negative prompt: ' + infotext + if 'Prompt:' not in infotext: + infotext = 'prompt: ' + infotext + + remaining = infotext + prompt = remaining[:infotext.lower().find('negative prompt:')] + remaining = remaining.replace(prompt, '') + if prompt.lower().startswith('prompt: '): + prompt = prompt[8:] + # debug(f'Prompt: {prompt}') + + params = ['steps:', 'seed:', 'width:', 'height:', 'sampler:', 'size:', 'cfg scale:'] # first param is one of those + param_idx = [remaining.lower().find(p) for p in params if p in remaining] + param_idx = min(param_idx) if len(param_idx) > 0 else 0 + negative = remaining[:param_idx] if param_idx > 0 else remaining + remaining = remaining.replace(negative, '') + if negative.lower().startswith('negative prompt: '): + negative = negative[16:] + # debug(f'Negative: {negative}') + + params = dict(re_param.findall(remaining)) + params['Prompt'] = prompt + params['Negative prompt'] = negative + for key, val in params.copy().items(): + val = unquote(val).strip(" ,\n\\n") + size = re_size.match(val) + if val.replace('.', '', 1).isdigit(): + params[key] = float(val) if '.' in val else int(val) + elif val == "True": + params[key] = True + elif val == "False": + params[key] = False + elif key == 'VAE' and val == 'TAESD': + params["Full quality"] = False + elif size is not None: + params[f"{key}-1"] = int(size.group(1)) + params[f"{key}-2"] = int(size.group(2)) + elif isinstance(params[key], str): + params[key] = val + debug(f'Param parsed: type={type(params[key])} {key}={params[key]} raw="{val}"') + + return params + + +mapping = [ + ('Backend', 'sd_backend'), + ('Model hash', 'sd_model_checkpoint'), + ('Refiner', 'sd_model_refiner'), + ('VAE', 'sd_vae'), + ('Parser', 'prompt_attention'), + ('Color correction', 'img2img_color_correction'), + # Samplers + ('Sampler Eta', 'scheduler_eta'), + ('Sampler ENSD', 'eta_noise_seed_delta'), + ('Sampler order', 'schedulers_solver_order'), + # Samplers diffusers + ('Sampler beta schedule', 'schedulers_beta_schedule'), + ('Sampler beta start', 'schedulers_beta_start'), + ('Sampler beta end', 'schedulers_beta_end'), + ('Sampler DPM solver', 'schedulers_dpm_solver'), + # Samplers original + ('Sampler brownian', 'schedulers_brownian_noise'), + ('Sampler discard', 'schedulers_discard_penultimate'), + ('Sampler dyn threshold', 'schedulers_use_thresholding'), + ('Sampler karras', 'schedulers_use_karras'), + ('Sampler low order', 'schedulers_use_loworder'), + ('Sampler quantization', 'enable_quantization'), + ('Sampler sigma', 'schedulers_sigma'), + ('Sampler sigma min', 's_min'), + ('Sampler sigma max', 's_max'), + ('Sampler sigma churn', 's_churn'), + ('Sampler sigma uncond', 's_min_uncond'), + ('Sampler sigma noise', 's_noise'), + ('Sampler sigma tmin', 's_tmin'), + ('Sampler ENSM', 'initial_noise_multiplier'), # img2img only + ('UniPC skip type', 'uni_pc_skip_type'), + ('UniPC variant', 'uni_pc_variant'), + # Token Merging + ('Mask weight', 'inpainting_mask_weight'), + ('ToMe', 'tome_ratio'), + ('ToDo', 'todo_ratio'), +] + + +if __name__ == '__main__': + import logging + log = logging.getLogger(__name__) + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s | %(message)s') + debug = log.info + + import sys + if len(sys.argv) > 1: + if os.path.exists(sys.argv[1]): + with open(sys.argv[1], 'r', encoding='utf8') as f: + parse(f.read()) + else: + parse(sys.argv[1]) diff --git a/modules/postprocessing.py b/modules/postprocessing.py index 4e307bec3..2288401df 100644 --- a/modules/postprocessing.py +++ b/modules/postprocessing.py @@ -4,7 +4,7 @@ from typing import List from PIL import Image -from modules import shared, images, devices, scripts, scripts_postprocessing, generation_parameters_copypaste +from modules import shared, images, devices, scripts, scripts_postprocessing, infotext from modules.shared import opts @@ -17,7 +17,7 @@ def run_postprocessing(extras_mode, image, image_folder: List[tempfile.NamedTemp image_ext = [] outputs = [] params = {} - infotext = '' + info = '' if extras_mode == 1: for img in image_folder: if isinstance(img, Image.Image): @@ -63,7 +63,7 @@ def run_postprocessing(extras_mode, image, image_folder: List[tempfile.NamedTemp processed_images = [] for image, name, ext in zip(image_data, image_names, image_ext): # pylint: disable=redefined-argument-from-local shared.log.debug(f'Process: image={image} {args}') - infotext = '' + info = '' if shared.state.interrupted: shared.log.debug('Postprocess interrupted') break @@ -73,27 +73,27 @@ def run_postprocessing(extras_mode, image, image_folder: List[tempfile.NamedTemp pp = scripts_postprocessing.PostprocessedImage(image.convert("RGB")) scripts.scripts_postproc.run(pp, args) geninfo, items = images.read_info_from_image(image) - params = generation_parameters_copypaste.parse_generation_parameters(geninfo) + params = infotext.parse(geninfo) for k, v in items.items(): pp.image.info[k] = v if 'parameters' in items: - infotext = items['parameters'] + ', ' - infotext = infotext + ", ".join([k if k == v else f'{k}: {generation_parameters_copypaste.quote(v)}' for k, v in pp.info.items() if v is not None]) - pp.image.info["postprocessing"] = infotext + info = items['parameters'] + ', ' + info = info + ", ".join([k if k == v else f'{k}: {info.quote(v)}' for k, v in pp.info.items() if v is not None]) + pp.image.info["postprocessing"] = info processed_images.append(pp.image) if save_output: if opts.use_original_name_batch and name is not None: forced_filename = os.path.splitext(os.path.basename(name))[0] - images.save_image(pp.image, path=outpath, extension=ext or opts.samples_format, info=infotext, short_filename=True, no_prompt=True, grid=False, pnginfo_section_name="extras", existing_info=pp.image.info, forced_filename=forced_filename) + images.save_image(pp.image, path=outpath, extension=ext or opts.samples_format, info=info, short_filename=True, no_prompt=True, grid=False, pnginfo_section_name="extras", existing_info=pp.image.info, forced_filename=forced_filename) else: - images.save_image(pp.image, path=outpath, extension=ext or opts.samples_format, info=infotext, short_filename=True, no_prompt=True, grid=False, pnginfo_section_name="extras", existing_info=pp.image.info) + images.save_image(pp.image, path=outpath, extension=ext or opts.samples_format, info=info, short_filename=True, no_prompt=True, grid=False, pnginfo_section_name="extras", existing_info=pp.image.info) if extras_mode != 2 or show_extras_results: outputs.append(pp.image) image.close() scripts.scripts_postproc.postprocess(processed_images, args) devices.torch_gc() - return outputs, infotext, params + return outputs, info, params def run_extras(extras_mode, resize_mode, image, image_folder, input_dir, output_dir, show_extras_results, gfpgan_visibility, codeformer_visibility, codeformer_weight, upscaling_resize, upscaling_resize_w, upscaling_resize_h, upscaling_crop, extras_upscaler_1, extras_upscaler_2, extras_upscaler_2_visibility, upscale_first: bool, save_output: bool = True): #pylint: disable=unused-argument diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 7f6b90d6d..bfe267e2e 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -130,8 +130,6 @@ def get_tokens(msg, prompt): def encode_prompts(pipe, p, prompts: list, negative_prompts: list, steps: int, clip_skip: typing.Optional[int] = None): - if getattr(p, 'hidiffusion', False) is True: - cache.clear() if 'StableDiffusion' not in pipe.__class__.__name__ and 'DemoFusion' not in pipe.__class__.__name__ and 'StableCascade' not in pipe.__class__.__name__: shared.log.warning(f"Prompt parser not supported: {pipe.__class__.__name__}") return diff --git a/modules/styles.py b/modules/styles.py index 511cfc425..d118800e8 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -6,7 +6,7 @@ import csv import json import time import random -from modules import files_cache, shared +from modules import files_cache, shared, infotext class Style(): @@ -132,12 +132,11 @@ def apply_styles_to_extra(p, style: Style): name_exclude = [ 'size', ] - from modules.generation_parameters_copypaste import parse_generation_parameters reference_style = get_reference_style() - extra = parse_generation_parameters(reference_style) if shared.opts.extra_network_reference else {} + extra = infotext.parse(reference_style) if shared.opts.extra_network_reference else {} style_extra = apply_wildcards_to_prompt(style.extra, [style.wildcards], silent=True) - extra.update(parse_generation_parameters(style_extra)) + extra.update(infotext.parse(style_extra)) extra.pop('Prompt', None) extra.pop('Negative prompt', None) fields = [] diff --git a/modules/ui_common.py b/modules/ui_common.py index ec1d7f505..9ad1e14c1 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -6,7 +6,7 @@ import platform import subprocess from functools import reduce import gradio as gr -from modules import call_queue, shared, prompt_parser, ui_sections, ui_symbols, ui_components, generation_parameters_copypaste, images, scripts, script_callbacks +from modules import call_queue, shared, prompt_parser, ui_sections, ui_symbols, ui_components, generation_parameters_copypaste, images, scripts, script_callbacks, infotext folder_symbol = ui_symbols.folder @@ -23,8 +23,8 @@ def update_generation_info(generation_info, html_info, img_index): generation_info = json.loads(generation_info) if img_index < 0 or img_index >= len(generation_info["infotexts"]): return html_info, generation_info - infotext = generation_info["infotexts"][img_index] - html_info_formatted = infotext_to_html(infotext) + info = generation_info["infotexts"][img_index] + html_info_formatted = infotext_to_html(info) return html_info, html_info_formatted except Exception: pass @@ -37,7 +37,7 @@ def plaintext_to_html(text): def infotext_to_html(text): - res = generation_parameters_copypaste.parse_generation_parameters(text) + res = infotext.parse(text) prompt = res.get('Prompt', '') negative = res.get('Negative prompt', '') res.pop('Prompt', None) @@ -169,7 +169,7 @@ def save_files(js_data, files, html_info, index): if (js_data is None or len(js_data) == 0) and image is not None and image.info is not None: info = image.info.pop('parameters', None) or image.info.pop('UserComment', None) geninfo, _ = images.read_info_from_image(image) - items = generation_parameters_copypaste.parse_generation_parameters(geninfo) + items = infotext.parse(geninfo) p = PObject(items) fullfn, txt_fullfn = images.save_image(image, shared.opts.outdir_save, "", seed=p.all_seeds[i], prompt=p.all_prompts[i], info=info, extension=shared.opts.samples_format, grid=is_grid, p=p) if fullfn is None: diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 4f0b48d26..8f93aef24 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -16,7 +16,7 @@ from collections import OrderedDict import gradio as gr from PIL import Image from starlette.responses import FileResponse, JSONResponse -from modules import paths, shared, scripts, files_cache, errors +from modules import paths, shared, scripts, files_cache, errors, infotext from modules.ui_components import ToolButton import modules.ui_symbols as symbols @@ -877,28 +877,26 @@ def create_ui(container, button_parent, tabname, skip_indexing = False): return ui_refresh_click(title) def ui_save_click(): - from modules import generation_parameters_copypaste filename = os.path.join(paths.data_path, "params.txt") if os.path.exists(filename): with open(filename, "r", encoding="utf8") as file: prompt = file.read() else: prompt = '' - params = generation_parameters_copypaste.parse_generation_parameters(prompt) + params = infotext.parse(prompt) res = show_details(text=None, img=None, desc=None, info=None, meta=None, parameters=None, description=None, prompt=None, negative=None, wildcards=None, params=params) return res def ui_quicksave_click(name): if name is None: return - from modules import generation_parameters_copypaste fn = os.path.join(paths.data_path, "params.txt") if os.path.exists(fn): with open(fn, "r", encoding="utf8") as file: prompt = file.read() else: prompt = '' - params = generation_parameters_copypaste.parse_generation_parameters(prompt) + params = infotext.parse(prompt) fn = os.path.join(shared.opts.styles_dir, os.path.splitext(name)[0] + '.json') prompt = params.get('Prompt', '') item = { From b8c6d63580b7573efa4400538a28e3ff2776d5e8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 6 Jun 2024 11:25:06 -0400 Subject: [PATCH 15/34] update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abbbc2441..04758a919 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,8 @@ - lower overhead on generate calls - cumulative fixes since the last release - add python version check for torch-directml -- improve metadata/infotext parser +- improve metadata/infotext parser + add `cli/image-exif.py` that can be used to view/extract metadata from images ## Update for 2024-06-02 From 4e1f8a2b711784636e27e3db2d9d5eb7ee7170a4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 6 Jun 2024 13:15:44 -0400 Subject: [PATCH 16/34] fix capitalization --- modules/infotext.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/infotext.py b/modules/infotext.py index ed569517d..285631f84 100644 --- a/modules/infotext.py +++ b/modules/infotext.py @@ -27,9 +27,9 @@ def parse(infotext): if not isinstance(infotext, str): return {} debug(f'Raw: {infotext}') - if 'negative prompt:' not in infotext: + if 'negative prompt:' not in infotext.lower(): infotext = 'negative prompt: ' + infotext - if 'Prompt:' not in infotext: + if 'prompt:' not in infotext.lower(): infotext = 'prompt: ' + infotext remaining = infotext From cc2d6618d6e5607a39bc0f0d4ae4373bbe83990e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 6 Jun 2024 13:27:41 -0400 Subject: [PATCH 17/34] fix linebreak strip --- modules/infotext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/infotext.py b/modules/infotext.py index 285631f84..425b8e633 100644 --- a/modules/infotext.py +++ b/modules/infotext.py @@ -52,7 +52,7 @@ def parse(infotext): params['Prompt'] = prompt params['Negative prompt'] = negative for key, val in params.copy().items(): - val = unquote(val).strip(" ,\n\\n") + val = unquote(val).strip(" ,\n").replace('\\\n', '') size = re_size.match(val) if val.replace('.', '', 1).isdigit(): params[key] = float(val) if '.' in val else int(val) From 6f019d8b80e7c3ae8c53cb0e4a6fb7d99c41b6e3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 6 Jun 2024 13:43:47 -0400 Subject: [PATCH 18/34] infotext handle break before params --- installer.py | 10 +++++++--- modules/infotext.py | 4 ++-- modules/update.py | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/installer.py b/installer.py index 328c57b29..3a89036ff 100644 --- a/installer.py +++ b/installer.py @@ -23,6 +23,7 @@ class Dot(dict): # dot notation access to dictionary attributes version = None +current_branch = None log = logging.getLogger("sd") debug = log.debug if os.environ.get('SD_INSTALL_DEBUG', None) is not None else lambda *args, **kwargs: None log_file = os.path.join(os.path.dirname(__file__), 'sdnext.log') @@ -294,6 +295,7 @@ def git(arg: str, folder: str = None, ignore: bool = False): # reattach as needed as head can get detached def branch(folder=None): + global current_branch # pylint: disable=global-statement # if args.experimental: # return None if not os.path.exists(os.path.join(folder or os.curdir, '.git')): @@ -318,17 +320,19 @@ def branch(folder=None): b = b.split('\n')[0].replace('*', '').strip() log.debug(f'Submodule: {folder} / {b}') git(f'checkout {b}', folder, ignore=True) + if folder is None: + current_branch = b return b # update git repository -def update(folder, current_branch = False, rebase = True): +def update(folder, keep_branch = False, rebase = True): try: git('config rebase.Autostash true') except Exception: pass arg = '--rebase --force' if rebase else '' - if current_branch: + if keep_branch: res = git(f'pull {arg}', folder) debug(f'Install update: folder={folder} args={arg} {res}') return res @@ -1003,7 +1007,7 @@ def check_version(offline=False, reset=True): # pylint: disable=unused-argument try: git('add .') git('stash') - update('.', current_branch=True) + update('.', keep_branch=True) # git('git stash pop') ver = git('log -1 --pretty=format:"%h %ad"') log.info(f'Upgraded to version: {ver}') diff --git a/modules/infotext.py b/modules/infotext.py index 425b8e633..7cc373580 100644 --- a/modules/infotext.py +++ b/modules/infotext.py @@ -32,7 +32,7 @@ def parse(infotext): if 'prompt:' not in infotext.lower(): infotext = 'prompt: ' + infotext - remaining = infotext + remaining = infotext.replace('\nSteps:', ' Steps:') prompt = remaining[:infotext.lower().find('negative prompt:')] remaining = remaining.replace(prompt, '') if prompt.lower().startswith('prompt: '): @@ -40,7 +40,7 @@ def parse(infotext): # debug(f'Prompt: {prompt}') params = ['steps:', 'seed:', 'width:', 'height:', 'sampler:', 'size:', 'cfg scale:'] # first param is one of those - param_idx = [remaining.lower().find(p) for p in params if p in remaining] + param_idx = [remaining.lower().find(p) for p in params if p in remaining.lower()] param_idx = min(param_idx) if len(param_idx) > 0 else 0 negative = remaining[:param_idx] if param_idx > 0 else remaining remaining = remaining.replace(negative, '') diff --git a/modules/update.py b/modules/update.py index 39157df07..220258add 100644 --- a/modules/update.py +++ b/modules/update.py @@ -56,7 +56,7 @@ def apply_update(update_rebase, update_submodules, update_extensions): if update_rebase: i.git('add .') i.git('stash') - res = i.update('.', current_branch=True, rebase=update_rebase) + res = i.update('.', keep_branch=True, rebase=update_rebase) html.append(res.replace('\n', '
')) except Exception as e: html.append(f'Error during repository upgrade: {e}') From a5a82d9108042264ada062d71a98715fb0b1349b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 6 Jun 2024 14:25:46 -0400 Subject: [PATCH 19/34] sync ui and core branches --- installer.py | 44 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/installer.py b/installer.py index 3a89036ff..532e62b25 100644 --- a/installer.py +++ b/installer.py @@ -293,9 +293,9 @@ def git(arg: str, folder: str = None, ignore: bool = False): log.debug(f'Git output: {txt}') return txt + # reattach as needed as head can get detached def branch(folder=None): - global current_branch # pylint: disable=global-statement # if args.experimental: # return None if not os.path.exists(os.path.join(folder or os.curdir, '.git')): @@ -320,8 +320,6 @@ def branch(folder=None): b = b.split('\n')[0].replace('*', '').strip() log.debug(f'Submodule: {folder} / {b}') git(f'checkout {b}', folder, ignore=True) - if folder is None: - current_branch = b return b @@ -950,9 +948,9 @@ def check_extensions(): return round(newest_all) -def get_version(): +def get_version(force=False): global version # pylint: disable=global-statement - if version is None: + if version is None or force: try: subprocess.run('git config log.showsignature false', stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True, check=True) except Exception: @@ -974,9 +972,41 @@ def get_version(): } except Exception: version = { 'app': 'sd.next', 'version': 'unknown' } + try: + cwd = os.getcwd() + os.chdir('extensions-builtin/sdnext-modernui') + res = subprocess.run('git rev-parse --abbrev-ref HEAD', stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True, check=True) + os.chdir(cwd) + branch_ui = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else '' + branch_ui = 'dev' if 'dev' in branch_ui else 'main' + version['ui'] = branch_ui + except Exception: + os.chdir(cwd) + version['ui'] = 'unknown' return version +def check_ui(ver): + if ver is None: + return + if ver['branch'] == ver['ui']: + return + log.warning(f'Branch mismatch: sdnext={ver["branch"]} ui={ver["ui"]}') + cwd = os.getcwd() + try: + os.chdir('extensions-builtin/sdnext-modernui') + git('checkout ' + ver['branch']) + os.chdir(cwd) + ver = get_version(force=True) + if ver['branch'] == ver['ui']: + log.info(f'Branch synchronized: {ver["branch"]}') + else: + log.error(f'Branch synchronize: sdnext={ver["branch"]} ui={ver["ui"]}') + except Exception as e: + log.error(f'Branch switch: {e}') + os.chdir(cwd) + + # check version of the main repo and optionally upgrade it def check_version(offline=False, reset=True): # pylint: disable=unused-argument if args.skip_all: @@ -984,9 +1014,11 @@ def check_version(offline=False, reset=True): # pylint: disable=unused-argument if not os.path.exists('.git'): log.warning('Not a git repository, all git operations are disabled') args.skip_git = True # pylint: disable=attribute-defined-outside-init - log.info(f'Version: {print_dict(get_version())}') + ver = get_version() + log.info(f'Version: {print_dict(ver)}') if args.version or args.skip_git: return + check_ui(ver) commit = git('rev-parse HEAD') global git_commit # pylint: disable=global-statement git_commit = commit[:7] From c8d6f03209cbace6e3b9b98658f396701a689701 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 6 Jun 2024 14:26:50 -0400 Subject: [PATCH 20/34] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04758a919..9bdcac483 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ - add python version check for torch-directml - improve metadata/infotext parser add `cli/image-exif.py` that can be used to view/extract metadata from images +- auto-synchronize modernui and core branches ## Update for 2024-06-02 From 5574833f0d04b94d17f39a053b67b60f1370e12d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 6 Jun 2024 20:41:10 -0400 Subject: [PATCH 21/34] refactor backend detection --- cli/image-exif.py | 4 +- .../Lora/extra_networks_lora.py | 2 +- extensions-builtin/Lora/lora_convert.py | 2 +- extensions-builtin/Lora/networks.py | 8 +-- .../Lora/ui_extra_networks_lora.py | 4 +- extensions-builtin/sdnext-modernui | 2 +- modules/face/__init__.py | 4 +- modules/face/faceid.py | 2 +- modules/hidiffusion/__init__.py | 2 + modules/interrogate.py | 6 +-- modules/ipadapter.py | 2 +- modules/layerdiffuse/__init__.py | 2 + modules/modeldata.py | 4 +- modules/pag/__init__.py | 2 + modules/postprocess/sdupscaler_model.py | 2 +- modules/processing.py | 18 +++---- modules/processing_class.py | 18 +++---- modules/processing_helpers.py | 6 +-- modules/processing_info.py | 8 +-- modules/prompt_parser.py | 4 +- modules/prompt_parser_diffusers.py | 38 ++++++-------- modules/sd_hijack.py | 4 +- modules/sd_hijack_hypertile.py | 4 +- modules/sd_models.py | 49 ++++++++++--------- modules/sd_samplers.py | 6 +-- modules/sd_samplers_common.py | 2 +- modules/sd_vae.py | 10 ++-- modules/shared.py | 40 +++++++-------- .../textual_inversion/textual_inversion.py | 8 +-- modules/ui.py | 2 +- modules/ui_common.py | 6 +-- modules/ui_control.py | 2 +- modules/ui_extra_networks.py | 2 +- modules/ui_extra_networks_checkpoints.py | 4 +- .../ui_extra_networks_textual_inversion.py | 4 +- modules/ui_img2img.py | 2 +- modules/ui_sections.py | 14 +++--- scripts/animatediff.py | 4 +- scripts/blipdiffusion.py | 2 +- scripts/demofusion.py | 2 +- scripts/differential_diffusion.py | 2 +- scripts/example.py | 2 +- scripts/image2video.py | 2 +- scripts/init_latents.py | 4 +- scripts/ipadapter.py | 4 +- scripts/kohya_hires_fix.py | 2 +- scripts/layerdiffuse.py | 2 +- scripts/ledits.py | 2 +- scripts/mixture_tiling.py | 2 +- scripts/mulan.py | 2 +- scripts/regional_prompting.py | 2 +- scripts/stablevideodiffusion.py | 2 +- scripts/t_gate.py | 2 +- scripts/text2video.py | 2 +- scripts/x_adapter.py | 2 +- 55 files changed, 171 insertions(+), 170 deletions(-) diff --git a/cli/image-exif.py b/cli/image-exif.py index 151a0df99..0aa7ffc55 100755 --- a/cli/image-exif.py +++ b/cli/image-exif.py @@ -9,7 +9,9 @@ from PIL import Image, ExifTags, TiffImagePlugin, PngImagePlugin from rich import print # pylint: disable=redefined-builtin -module_spec = importlib.util.spec_from_file_location('infotext', os.path.join('modules', 'infotext.py')) +module_file = os.path.abspath(__file__) +module_dir = os.path.dirname(module_file) +module_spec = importlib.util.spec_from_file_location('infotext', os.path.join(module_dir, '..', 'modules', 'infotext.py')) infotext = importlib.util.module_from_spec(module_spec) module_spec.loader.exec_module(infotext) diff --git a/extensions-builtin/Lora/extra_networks_lora.py b/extensions-builtin/Lora/extra_networks_lora.py index 22d8265d7..75eb31825 100644 --- a/extensions-builtin/Lora/extra_networks_lora.py +++ b/extensions-builtin/Lora/extra_networks_lora.py @@ -104,7 +104,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): self.active = False def deactivate(self, p): - if shared.backend == shared.Backend.DIFFUSERS and hasattr(shared.sd_model, "unload_lora_weights") and hasattr(shared.sd_model, "text_encoder"): + if shared.native and hasattr(shared.sd_model, "unload_lora_weights") and hasattr(shared.sd_model, "text_encoder"): if 'CLIP' in shared.sd_model.text_encoder.__class__.__name__ and not (shared.compiled_model_state is not None and shared.compiled_model_state.is_compiled is True): if shared.opts.lora_fuse_diffusers: shared.sd_model.unfuse_lora() diff --git a/extensions-builtin/Lora/lora_convert.py b/extensions-builtin/Lora/lora_convert.py index 1c08d8931..827f97e3d 100644 --- a/extensions-builtin/Lora/lora_convert.py +++ b/extensions-builtin/Lora/lora_convert.py @@ -106,7 +106,7 @@ def make_unet_conversion_map() -> Dict[str, str]: class KeyConvert: def __init__(self): - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: self.converter = self.original self.is_sd2 = 'model_transformer_resblocks' in shared.sd_model.network_layer_mapping else: diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index 5c4e539ec..d36809d47 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -47,7 +47,7 @@ convert_diffusers_name_to_compvis = lora_convert.convert_diffusers_name_to_compv def assign_network_names_to_compvis_modules(sd_model): network_layer_mapping = {} - if shared.backend == shared.Backend.DIFFUSERS: + if shared.native: if not hasattr(shared.sd_model, 'text_encoder') or not hasattr(shared.sd_model, 'unet'): return for name, module in shared.sd_model.text_encoder.named_modules(): @@ -85,7 +85,7 @@ def load_diffusers(name, network_on_disk, lora_scale=1.0) -> network.Network: shared.log.debug(f'LoRA load: name="{name}" file="{network_on_disk.filename}" type=diffusers {"cached" if cached else ""} fuse={shared.opts.lora_fuse_diffusers}') if cached is not None: return cached - if shared.backend != shared.Backend.DIFFUSERS: + if shared.native: return None shared.sd_model.load_lora_weights(network_on_disk.filename) if shared.opts.lora_fuse_diffusers: @@ -195,9 +195,9 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No try: if recompile_model: shared.compiled_model_state.lora_model.append(f"{name}:{te_multipliers[i] if te_multipliers else 1.0}") - if shared.backend == shared.Backend.DIFFUSERS and shared.opts.lora_force_diffusers: # OpenVINO only works with Diffusers LoRa loading + if shared.native and shared.opts.lora_force_diffusers: # OpenVINO only works with Diffusers LoRa loading net = load_diffusers(name, network_on_disk, lora_scale=te_multipliers[i] if te_multipliers else 1.0) - elif shared.backend == shared.Backend.DIFFUSERS and network_overrides.check_override(shorthash): + elif shared.native and network_overrides.check_override(shorthash): net = load_diffusers(name, network_on_disk, lora_scale=te_multipliers[i] if te_multipliers else 1.0) else: net = load_network(name, network_on_disk) diff --git a/extensions-builtin/Lora/ui_extra_networks_lora.py b/extensions-builtin/Lora/ui_extra_networks_lora.py index 1c172ddbf..d224f4c67 100644 --- a/extensions-builtin/Lora/ui_extra_networks_lora.py +++ b/extensions-builtin/Lora/ui_extra_networks_lora.py @@ -19,10 +19,10 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage): try: # path, _ext = os.path.splitext(l.filename) name = os.path.splitext(os.path.relpath(l.filename, shared.cmd_opts.lora_dir))[0] - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: if l.sd_version == network.SdVersion.SDXL: return None - elif shared.backend == shared.Backend.DIFFUSERS: + elif shared.native: if shared.sd_model_type == 'none': # return all when model is not loaded pass elif shared.sd_model_type == 'sdxl': diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 8afbad75d..cc2e7ee98 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 8afbad75d6cd238270111ec77ff19b567855d8bd +Subproject commit cc2e7ee980be3efaa514c68fac2b715cddfbc072 diff --git a/modules/face/__init__.py b/modules/face/__init__.py index 289bdb8e2..d1ded8c37 100644 --- a/modules/face/__init__.py +++ b/modules/face/__init__.py @@ -12,7 +12,7 @@ class Script(scripts.Script): return 'Face' def show(self, is_img2img): - return True if shared.backend == shared.Backend.DIFFUSERS else False + return True if shared.native else False def load_images(self, files): init_images = [] @@ -90,7 +90,7 @@ class Script(scripts.Script): return [mode, gallery, ip_model, ip_override, ip_cache, ip_strength, ip_structure, id_strength, id_conditioning, id_cache, pm_trigger, pm_strength, pm_start, fs_cache] def run(self, p: processing.StableDiffusionProcessing, mode, input_images, ip_model, ip_override, ip_cache, ip_strength, ip_structure, id_strength, id_conditioning, id_cache, pm_trigger, pm_strength, pm_start, fs_cache): # pylint: disable=arguments-differ, unused-argument - if shared.backend != shared.Backend.DIFFUSERS: + if not shared.native: return None if mode == 'None': return None diff --git a/modules/face/faceid.py b/modules/face/faceid.py index f400f038a..ec25e2f3d 100644 --- a/modules/face/faceid.py +++ b/modules/face/faceid.py @@ -70,7 +70,7 @@ def face_id( if shared.opts.cuda_compile_backend == 'none': sd_models.apply_token_merging(p.sd_model) - sd_hijack_freeu.apply_freeu(p, shared.backend == shared.Backend.ORIGINAL) + sd_hijack_freeu.apply_freeu(p, not shared.native) script_callbacks.before_process_callback(p) diff --git a/modules/hidiffusion/__init__.py b/modules/hidiffusion/__init__.py index e8b5f0fd0..2ef4c032a 100644 --- a/modules/hidiffusion/__init__.py +++ b/modules/hidiffusion/__init__.py @@ -6,6 +6,8 @@ from modules.hidiffusion import hidiffusion def apply_hidiffusion(p, model_type): + if not shared.native: + return if model_type not in ['sd', 'sdxl'] and p.hidiffusion: shared.log.warning(f'HiDiffusion: class={shared.sd_model.__class__.__name__} not supported') return diff --git a/modules/interrogate.py b/modules/interrogate.py index ae4cd2926..eed43c773 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -165,7 +165,7 @@ class InterrogateModels: res = "" shared.state.begin('Interrogate') try: - if shared.backend == shared.Backend.ORIGINAL and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): + if not shared.native and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): lowvram.send_everything_to_cpu() devices.torch_gc() self.load() @@ -269,7 +269,7 @@ def interrogate(image, mode, caption=None): def interrogate_image(image, model, mode): shared.state.begin('Interrogate') try: - if shared.backend == shared.Backend.ORIGINAL and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): + if not shared.native and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): lowvram.send_everything_to_cpu() devices.torch_gc() load_interrogator(model) @@ -297,7 +297,7 @@ def interrogate_batch(batch_files, batch_folder, batch_str, model, mode, write): shared.state.begin('Batch interrogate') prompts = [] try: - if shared.backend == shared.Backend.ORIGINAL and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): + if not shared.native and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): lowvram.send_everything_to_cpu() devices.torch_gc() load_interrogator(model) diff --git a/modules/ipadapter.py b/modules/ipadapter.py index dccfde622..dd073e722 100644 --- a/modules/ipadapter.py +++ b/modules/ipadapter.py @@ -138,7 +138,7 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt # init code if pipe is None: return False - if shared.backend != shared.Backend.DIFFUSERS: + if not shared.native: shared.log.warning('IP adapter: not in diffusers mode') return False if len(adapter_images) == 0: diff --git a/modules/layerdiffuse/__init__.py b/modules/layerdiffuse/__init__.py index 929ec7418..368790c19 100644 --- a/modules/layerdiffuse/__init__.py +++ b/modules/layerdiffuse/__init__.py @@ -41,6 +41,8 @@ def apply_layerdiffuse_sdxl_conv(pipeline): def apply_layerdiffuse(): + if not shared.native: + return try: if shared.sd_model_type == 'sd': shared.log.info(f'LayerDiffuse: class={shared.sd_model.__class__.__name__}') diff --git a/modules/modeldata.py b/modules/modeldata.py index 600763635..c7d25a0ac 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -81,7 +81,7 @@ class Shared(sys.modules[__name__].__class__): if modules.sd_models.model_data.sd_model is None: model_type = 'none' return model_type - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: model_type = 'ldm' elif "StableDiffusionXL" in self.sd_model.__class__.__name__: model_type = 'sdxl' @@ -110,7 +110,7 @@ class Shared(sys.modules[__name__].__class__): if modules.sd_models.model_data.sd_refiner is None: model_type = 'none' return model_type - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: model_type = 'ldm' elif "StableDiffusionXL" in self.sd_refiner.__class__.__name__: model_type = 'sdxl' diff --git a/modules/pag/__init__.py b/modules/pag/__init__.py index 2b2b84502..8e43bb30b 100644 --- a/modules/pag/__init__.py +++ b/modules/pag/__init__.py @@ -11,6 +11,8 @@ orig_pipeline = None def apply(p: processing.StableDiffusionProcessing): # pylint: disable=arguments-differ global orig_pipeline # pylint: disable=global-statement c = shared.sd_model.__class__ if shared.sd_loaded else None + if not shared.native: + return None if p.pag_scale == 0: unapply() return None diff --git a/modules/postprocess/sdupscaler_model.py b/modules/postprocess/sdupscaler_model.py index 0d73b8fa9..0a3106289 100644 --- a/modules/postprocess/sdupscaler_model.py +++ b/modules/postprocess/sdupscaler_model.py @@ -8,7 +8,7 @@ class UpscalerSD(Upscaler): def __init__(self, dirname): # pylint: disable=super-init-not-called self.name = "SDUpscale" self.user_path = dirname - if shared.backend != shared.Backend.DIFFUSERS: + if not shared.native: super().__init__() return self.scalers = [ diff --git a/modules/processing.py b/modules/processing.py index e0570d42a..4782c8b4d 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -161,7 +161,7 @@ def process_images(p: StableDiffusionProcessing) -> Processed: pag.apply(p) if shared.opts.cuda_compile_backend == 'none': sd_models.apply_token_merging(p.sd_model) - sd_hijack_freeu.apply_freeu(p, shared.backend == shared.Backend.ORIGINAL) + sd_hijack_freeu.apply_freeu(p, not shared.native) if p.width is not None: p.width = 8 * int(p.width / 8) @@ -247,7 +247,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: else: assert p.prompt is not None - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: import modules.sd_hijack # pylint: disable=redefined-outer-name modules.sd_hijack.model_hijack.apply_circular(p.tiling) modules.sd_hijack.model_hijack.clear_comments() @@ -256,7 +256,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: output_images = [] process_init(p) - if os.path.exists(shared.opts.embeddings_dir) and not p.do_not_reload_embeddings and shared.backend == shared.Backend.ORIGINAL: + if os.path.exists(shared.opts.embeddings_dir) and not p.do_not_reload_embeddings and not shared.native: modules.sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings(force_reload=False) if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner): p.scripts.process(p) @@ -264,7 +264,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: def infotext(_inxex=0): # dummy function overriden if there are iterations return '' - ema_scope_context = p.sd_model.ema_scope if shared.backend == shared.Backend.ORIGINAL else nullcontext + ema_scope_context = p.sd_model.ema_scope if not shared.native else nullcontext shared.state.job_count = p.n_iter with devices.inference_context(), ema_scope_context(): t0 = time.time() @@ -283,7 +283,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: shared.log.debug(f'Process interrupted: {n+1}/{p.n_iter}') break - if shared.backend == shared.Backend.DIFFUSERS: + if shared.native: from modules import ipadapter ipadapter.apply(shared.sd_model, p) p.prompts = p.all_prompts[n * p.batch_size:(n+1) * p.batch_size] @@ -304,10 +304,10 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner): x_samples_ddim = p.scripts.process_images(p) if x_samples_ddim is None: - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: from modules.processing_original import process_original x_samples_ddim = process_original(p) - elif shared.backend == shared.Backend.DIFFUSERS: + elif shared.native: from modules.processing_diffusers import process_diffusers x_samples_ddim = process_diffusers(p) else: @@ -316,7 +316,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if not shared.opts.keep_incomplete and shared.state.interrupted: x_samples_ddim = [] - if shared.backend == shared.Backend.ORIGINAL and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): + if not shared.native and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): lowvram.send_everything_to_cpu() devices.torch_gc() if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner): @@ -407,7 +407,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if shared.opts.grid_save: images.save_image(grid, p.outpath_grids, "", p.all_seeds[0], p.all_prompts[0], shared.opts.grid_format, info=infotext(-1), p=p, grid=True, suffix="-grid") # main save grid - if shared.backend == shared.Backend.DIFFUSERS: + if shared.native: from modules import ipadapter ipadapter.unapply(shared.sd_model) diff --git a/modules/processing_class.py b/modules/processing_class.py index 7055c2ce1..9f8ac7792 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -215,7 +215,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.script_args = [] def init(self, all_prompts=None, all_seeds=None, all_subseeds=None): - if shared.backend == shared.Backend.DIFFUSERS: + if shared.native: shared.sd_model = sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) self.width = self.width or 512 self.height = self.height or 512 @@ -252,7 +252,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.hr_upscale_to_y = self.hr_resize_y self.truncate_x = (self.hr_upscale_to_x - target_w) // 8 self.truncate_y = (self.hr_upscale_to_y - target_h) // 8 - if shared.backend == shared.Backend.ORIGINAL: # diffusers are handled in processing_diffusers + if not shared.native: # diffusers are handled in processing_diffusers if (self.hr_upscale_to_x == self.width and self.hr_upscale_to_y == self.height) or upscaler is None or upscaler == 'None': # special case: the user has chosen to do nothing self.is_hr_pass = False return @@ -303,9 +303,9 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): self.script_args = [] def init(self, all_prompts=None, all_seeds=None, all_subseeds=None): - if shared.backend == shared.Backend.DIFFUSERS and getattr(self, 'image_mask', None) is not None: + if shared.native and getattr(self, 'image_mask', None) is not None: shared.sd_model = sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.INPAINTING) - elif shared.backend == shared.Backend.DIFFUSERS and getattr(self, 'init_images', None) is not None: + elif shared.native and getattr(self, 'init_images', None) is not None: shared.sd_model = sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) if all_prompts is not None: @@ -317,7 +317,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): if self.sampler_name == "PLMS": self.sampler_name = 'UniPC' - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: self.sampler = sd_samplers.create_sampler(self.sampler_name, self.sd_model) if hasattr(self.sampler, "initialize"): self.sampler.initialize(self) @@ -331,7 +331,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): if self.image_mask is not None: if type(self.image_mask) == list: self.image_mask = self.image_mask[0] - if shared.backend == shared.Backend.ORIGINAL: # original way of processing mask + if not shared.native: # original way of processing mask self.image_mask = processing_helpers.create_binary_mask(self.image_mask) if self.inpainting_mask_invert: self.image_mask = ImageOps.invert(self.image_mask) @@ -341,7 +341,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): np_mask = cv2.GaussianBlur(np_mask, (kernel_size, 1), self.mask_blur) np_mask = cv2.GaussianBlur(np_mask, (1, kernel_size), self.mask_blur) self.image_mask = Image.fromarray(np_mask) - elif shared.backend == shared.Backend.DIFFUSERS: + elif shared.native: if 'control' in self.ops: self.image_mask = masking.run_mask(input_image=self.init_images, input_mask=self.image_mask, return_type='Grayscale', invert=self.inpainting_mask_invert==1) # blur/padding are handled in masking module else: @@ -411,9 +411,9 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): self.overlay_images = self.overlay_images * self.batch_size if self.color_corrections is not None and len(self.color_corrections) == 1: self.color_corrections = self.color_corrections * self.batch_size - if shared.backend == shared.Backend.DIFFUSERS: + if shared.native: return # we've already set self.init_images and self.mask and we dont need any more processing - elif shared.backend == shared.Backend.ORIGINAL: + elif not shared.native: self.init_images = [np.moveaxis((np.array(image).astype(np.float32) / 255.0), 2, 0) for image in self.init_images] if len(self.init_images) == 1: batch_images = np.expand_dims(self.init_images[0], axis=0).repeat(self.batch_size, axis=0) diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index b093e71fd..30199dd98 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -35,9 +35,9 @@ def apply_color_correction(correction, original_image): def apply_overlay(image: Image, paste_loc, index, overlays): - debug(f'Apply overlay: image={image} loc={paste_loc} index={index} overlays={overlays}') if overlays is None or index >= len(overlays): return image + debug(f'Apply overlay: image={image} loc={paste_loc} index={index} overlays={overlays}') overlay = overlays[index] if paste_loc is not None: x, y, w, h = paste_loc @@ -321,7 +321,7 @@ def img2img_image_conditioning(p, source_image, latent_image, image_mask=None): # HACK: Using introspection as the Depth2Image model doesn't appear to uniquely # identify itself with a field common to all models. The conditioning_key is also hybrid. - if shared.backend == shared.Backend.DIFFUSERS: + if shared.native: return diffusers_image_conditioning(source_image, latent_image, image_mask) if isinstance(p.sd_model, LatentDepth2ImageDiffusion): return depth2img_image_conditioning(source_image) @@ -346,7 +346,7 @@ def validate_sample(tensor): sample = tensor else: shared.log.warning(f'Unknown sample type: {type(tensor)}') - sample = 255.0 * np.moveaxis(sample, 0, 2) if shared.backend == shared.Backend.ORIGINAL else 255.0 * sample + sample = 255.0 * np.moveaxis(sample, 0, 2) if not shared.native else 255.0 * sample with warnings.catch_warnings(record=True) as w: cast = sample.astype(np.uint8) if len(w) > 0: diff --git a/modules/processing_info.py b/modules/processing_info.py index 809d87115..fb376627f 100644 --- a/modules/processing_info.py +++ b/modules/processing_info.py @@ -4,7 +4,7 @@ from modules import shared, sd_samplers_common, sd_vae, generation_parameters_co from modules.processing_class import StableDiffusionProcessing -if shared.backend == shared.Backend.ORIGINAL: +if not shared.native: from modules import sd_hijack else: sd_hijack = None @@ -57,7 +57,7 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No "Styles": "; ".join(p.styles) if p.styles is not None and len(p.styles) > 0 else None, "Tiling": p.tiling if p.tiling else None, # sdnext - "Backend": 'Diffusers' if shared.backend == shared.Backend.DIFFUSERS else 'Original', + "Backend": 'Diffusers' if shared.native else 'Original', "App": 'SD.Next', "Version": git_commit, "Comment": comment, @@ -109,12 +109,12 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No args["Sampler ENSD"] = shared.opts.eta_noise_seed_delta if shared.opts.eta_noise_seed_delta != 0 and sd_samplers_common.is_sampler_using_eta_noise_seed_delta(p) else None args["Sampler ENSM"] = p.initial_noise_multiplier if getattr(p, 'initial_noise_multiplier', 1.0) != 1.0 else None args['Sampler order'] = shared.opts.schedulers_solver_order if shared.opts.schedulers_solver_order != shared.opts.data_labels.get('schedulers_solver_order').default else None - if shared.backend == shared.Backend.DIFFUSERS: + if shared.native: args['Sampler beta schedule'] = shared.opts.schedulers_beta_schedule if shared.opts.schedulers_beta_schedule != shared.opts.data_labels.get('schedulers_beta_schedule').default else None args['Sampler beta start'] = shared.opts.schedulers_beta_start if shared.opts.schedulers_beta_start != shared.opts.data_labels.get('schedulers_beta_start').default else None args['Sampler beta end'] = shared.opts.schedulers_beta_end if shared.opts.schedulers_beta_end != shared.opts.data_labels.get('schedulers_beta_end').default else None args['Sampler DPM solver'] = shared.opts.schedulers_dpm_solver if shared.opts.schedulers_dpm_solver != shared.opts.data_labels.get('schedulers_dpm_solver').default else None - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: args['Sampler brownian'] = shared.opts.schedulers_brownian_noise if shared.opts.schedulers_brownian_noise != shared.opts.data_labels.get('schedulers_brownian_noise').default else None args['Sampler discard'] = shared.opts.schedulers_discard_penultimate if shared.opts.schedulers_discard_penultimate != shared.opts.data_labels.get('schedulers_discard_penultimate').default else None args['Sampler dyn threshold'] = shared.opts.schedulers_use_thresholding if shared.opts.schedulers_use_thresholding != shared.opts.data_labels.get('schedulers_use_thresholding').default else None diff --git a/modules/prompt_parser.py b/modules/prompt_parser.py index 2d164cd59..2a71d5053 100644 --- a/modules/prompt_parser.py +++ b/modules/prompt_parser.py @@ -14,7 +14,7 @@ from typing import List import lark import torch from compel import Compel -from modules.shared import opts, log, backend, Backend +from modules.shared import opts, log, native # a prompt like this: "fantasy landscape with a [mountain:lake:0.25] and [an oak:a christmas tree:0.75][ in foreground::0.6][ in background:0.25] [shoddy:masterful:0.5]" # will be represented with prompt_schedule like this (assuming steps=100): @@ -326,7 +326,7 @@ def parse_prompt_attention(text): whitespace = '' else: re_attention = re_attention_v1 - if backend == Backend.DIFFUSERS: + if native: text = text.replace('\n', ' BREAK ') else: text = text.replace('\n', ' ') diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index bfe267e2e..5fe559353 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -104,7 +104,7 @@ def get_prompt_schedule(prompt, steps): def get_tokens(msg, prompt): global token_dict, token_type # pylint: disable=global-statement - if shared.backend != shared.Backend.DIFFUSERS: + if not shared.native: return if shared.sd_loaded and hasattr(shared.sd_model, 'tokenizer') and shared.sd_model.tokenizer is not None: if token_dict is None or token_type != shared.sd_model_type: @@ -133,7 +133,7 @@ def encode_prompts(pipe, p, prompts: list, negative_prompts: list, steps: int, c if 'StableDiffusion' not in pipe.__class__.__name__ and 'DemoFusion' not in pipe.__class__.__name__ and 'StableCascade' not in pipe.__class__.__name__: shared.log.warning(f"Prompt parser not supported: {pipe.__class__.__name__}") return - elif prompts == cache.get('prompts', None) and negative_prompts == cache.get('negative_prompts', None) and clip_skip == cache.get('clip_skip', None) and cache.get('model_type', None) == shared.sd_model_type: + elif prompts == cache.get('prompts', None) and negative_prompts == cache.get('negative_prompts', None) and clip_skip == cache.get('clip_skip', None) and cache.get('model_type', None) == shared.sd_model_type and steps == cache.get('steps', None): p.prompt_embeds = cache.get('prompt_embeds', None) p.positive_pooleds = cache.get('positive_pooleds', None) p.negative_embeds = cache.get('negative_embeds', None) @@ -154,36 +154,28 @@ def encode_prompts(pipe, p, prompts: list, negative_prompts: list, steps: int, c for i in range(max(len(positive_schedule), len(negative_schedule))): positive_prompt = positive_schedule[i % len(positive_schedule)] negative_prompt = negative_schedule[i % len(negative_schedule)] - if cache.get('model_type', None) != shared.sd_model_type: - cache[positive_prompt + negative_prompt] = None - results = None - elif clip_skip == cache.get('clip_skip', None): - results = cache.get(positive_prompt + negative_prompt, None) - else: - results = None - - if results is None: - results = get_weighted_text_embeddings(pipe, positive_prompt, negative_prompt, clip_skip) - cache[positive_prompt + negative_prompt] = results - - prompt_embed, positive_pooled, negative_embed, negative_pooled = results + prompt_embed, positive_pooled, negative_embed, negative_pooled = get_weighted_text_embeddings(pipe, positive_prompt, negative_prompt, clip_skip) if prompt_embed is not None: p.prompt_embeds.append(torch.cat([prompt_embed] * len(prompts), dim=0)) - cache['prompt_embeds'] = p.prompt_embeds if negative_embed is not None: p.negative_embeds.append(torch.cat([negative_embed] * len(negative_prompts), dim=0)) - cache['negative_embeds'] = p.negative_embeds if positive_pooled is not None: p.positive_pooleds.append(torch.cat([positive_pooled] * len(prompts), dim=0)) - cache['positive_pooleds'] = p.positive_pooleds if negative_pooled is not None: p.negative_pooleds.append(torch.cat([negative_pooled] * len(negative_prompts), dim=0)) - cache['negative_pooleds'] = p.negative_pooleds - cache['prompts'] = prompts - cache['negative_prompts'] = negative_prompts - cache['clip_skip'] = clip_skip - cache['model_type'] = shared.sd_model_type + cache.update({ + 'prompt_embeds': p.prompt_embeds, + 'negative_embeds': p.negative_embeds, + 'positive_pooleds': p.positive_pooleds, + 'negative_pooleds': p.negative_pooleds, + 'scheduled_prompt': p.scheduled_prompt, + 'prompts': prompts, + 'negative_prompts': negative_prompts, + 'clip_skip': clip_skip, + 'steps': steps, + 'model_type': shared.sd_model_type + }) if debug_enabled: get_tokens('positive', prompts[0]) get_tokens('negative', negative_prompts[0]) diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 2417c67ab..b811f33bf 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -175,7 +175,7 @@ class StableDiffusionModelHijack: if m.cond_stage_key == "edit": sd_hijack_unet.hijack_ddpm_edit() - if "Model" in shared.opts.ipex_optimize and shared.backend == shared.Backend.ORIGINAL: + if "Model" in shared.opts.ipex_optimize and not shared.native: try: import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import m.model.eval() @@ -185,7 +185,7 @@ class StableDiffusionModelHijack: except Exception as err: shared.log.warning(f"IPEX Optimize not supported: {err}") - if "Model" in shared.opts.cuda_compile and shared.opts.cuda_compile_backend != 'none' and shared.backend == shared.Backend.ORIGINAL: + if "Model" in shared.opts.cuda_compile and shared.opts.cuda_compile_backend != 'none' and not shared.native: try: import logging shared.log.info(f"Compiling pipeline={m.model.__class__.__name__} mode={shared.opts.cuda_compile_backend}") diff --git a/modules/sd_hijack_hypertile.py b/modules/sd_hijack_hypertile.py index 296ceefc5..dbf977b8d 100644 --- a/modules/sd_hijack_hypertile.py +++ b/modules/sd_hijack_hypertile.py @@ -186,7 +186,7 @@ def context_hypertile_vae(p): error_reported = False height, width = p.height, p.width max_h, max_w = 0, 0 - vae = getattr(p.sd_model, "vae", None) if shared.backend == shared.Backend.DIFFUSERS else getattr(p.sd_model, "first_stage_model", None) + vae = getattr(p.sd_model, "vae", None) if shared.native else getattr(p.sd_model, "first_stage_model", None) if height % 8 != 0 or width % 8 != 0: log.warning(f'Hypertile VAE disabled: width={width} height={height} are not divisible by 8') return nullcontext() @@ -211,7 +211,7 @@ def context_hypertile_unet(p): error_reported = False height, width = p.height, p.width max_h, max_w = 0, 0 - unet = getattr(p.sd_model, "unet", None) if shared.backend == shared.Backend.DIFFUSERS else getattr(p.sd_model.model, "diffusion_model", None) + unet = getattr(p.sd_model, "unet", None) if shared.native else getattr(p.sd_model.model, "diffusion_model", None) if height % 8 != 0 or width % 8 != 0: log.warning(f'Hypertile UNet disabled: width={width} height={height} are not divisible by 8') return nullcontext() diff --git a/modules/sd_models.py b/modules/sd_models.py index 826c8d708..f9f89f189 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -127,7 +127,7 @@ def setup_model(): list_models() sd_hijack_accelerate.hijack_hfhub() # sd_hijack_accelerate.hijack_torch_conv() - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: enable_midas_autodownload() @@ -144,19 +144,19 @@ def list_models(): global checkpoints_list # pylint: disable=global-statement checkpoints_list.clear() checkpoint_aliases.clear() - ext_filter = [".safetensors"] if shared.opts.sd_disable_ckpt or shared.backend == shared.Backend.DIFFUSERS else [".ckpt", ".safetensors"] + ext_filter = [".safetensors"] if shared.opts.sd_disable_ckpt or shared.native else [".ckpt", ".safetensors"] model_list = list(modelloader.load_models(model_path=model_path, model_url=None, command_path=shared.opts.ckpt_dir, ext_filter=ext_filter, download_name=None, ext_blacklist=[".vae.ckpt", ".vae.safetensors"])) for filename in sorted(model_list, key=str.lower): checkpoint_info = CheckpointInfo(filename) if checkpoint_info.name is not None: checkpoint_info.register() - if shared.backend == shared.Backend.DIFFUSERS: + if shared.native: for repo in modelloader.load_diffusers_models(clear=True): checkpoint_info = CheckpointInfo(repo['name'], sha=repo['hash']) if checkpoint_info.name is not None: checkpoint_info.register() if shared.cmd_opts.ckpt is not None: - if not os.path.exists(shared.cmd_opts.ckpt) and shared.backend == shared.Backend.ORIGINAL: + if not os.path.exists(shared.cmd_opts.ckpt) and not shared.native: if shared.cmd_opts.ckpt.lower() != "none": shared.log.warning(f"Requested checkpoint not found: {shared.cmd_opts.ckpt}") else: @@ -414,7 +414,7 @@ def get_checkpoint_state_dict(checkpoint_info: CheckpointInfo, timer): checkpoints_loaded.move_to_end(checkpoint_info, last=True) # FIFO -> LRU cache return checkpoints_loaded[checkpoint_info] res = read_state_dict(checkpoint_info.filename) - if shared.opts.sd_checkpoint_cache > 0 and shared.backend == shared.Backend.ORIGINAL: + if shared.opts.sd_checkpoint_cache > 0 and not shared.native: # cache newly loaded model checkpoints_loaded[checkpoint_info] = res # clean up cache if limit is reached @@ -536,6 +536,7 @@ def change_backend(): shared.log.warning('Full server restart required to apply all changes') unload_model_weights() shared.backend = shared.Backend.ORIGINAL if shared.opts.sd_backend == 'original' else shared.Backend.DIFFUSERS + shared.native = shared.backend == shared.Backend.DIFFUSERS checkpoints_loaded.clear() from modules.sd_samplers import list_samplers list_samplers(shared.backend) @@ -564,29 +565,29 @@ def detect_pipeline(f: str, op: str = 'model', warning=True): # elif size < 0: # unknown # guess = 'Stable Diffusion 2B' elif size >= 5791 and size <= 5799: # 5795 - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: warn(f'Model detected as SD-XL refiner model, but attempting to load using backend=original: {op}={f} size={size} MB') if op == 'model': warn(f'Model detected as SD-XL refiner model, but attempting to load a base model: {op}={f} size={size} MB') guess = 'Stable Diffusion XL Refiner' elif (size >= 6611 and size <= 7220): # 6617, HassakuXL is 6776, monkrenRealisticINT_v10 is 7217 - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: warn(f'Model detected as SD-XL base model, but attempting to load using backend=original: {op}={f} size={size} MB') guess = 'Stable Diffusion XL' elif size >= 3361 and size <= 3369: # 3368 - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: warn(f'Model detected as SD upscale model, but attempting to load using backend=original: {op}={f} size={size} MB') guess = 'Stable Diffusion Upscale' elif size >= 4891 and size <= 4899: # 4897 - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: warn(f'Model detected as SD XL inpaint model, but attempting to load using backend=original: {op}={f} size={size} MB') guess = 'Stable Diffusion XL Inpaint' elif size >= 9791 and size <= 9799: # 9794 - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: warn(f'Model detected as SD XL instruct pix2pix model, but attempting to load using backend=original: {op}={f} size={size} MB') guess = 'Stable Diffusion XL Instruct' elif size > 3138 and size < 3142: #3140 - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: warn(f'Model detected as Segmind Vega model, but attempting to load using backend=original: {op}={f} size={size} MB') guess = 'Stable Diffusion XL' # guess by name @@ -597,29 +598,29 @@ def detect_pipeline(f: str, op: str = 'model', warning=True): guess = 'Latent Consistency Model' """ if 'instaflow' in f.lower(): - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: warn(f'Model detected as InstaFlow model, but attempting to load using backend=original: {op}={f} size={size} MB') guess = 'InstaFlow' if 'segmoe' in f.lower(): - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: warn(f'Model detected as SegMoE model, but attempting to load using backend=original: {op}={f} size={size} MB') guess = 'SegMoE' if 'hunyuandit' in f.lower(): - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: warn(f'Model detected as Tenecent HunyuanDiT model, but attempting to load using backend=original: {op}={f} size={size} MB') guess = 'HunyuanDiT' if 'pixart-xl' in f.lower(): - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: warn(f'Model detected as PixArt Alpha model, but attempting to load using backend=original: {op}={f} size={size} MB') guess = 'PixArt-Alpha' if 'stable-cascade' in f.lower() or 'stablecascade' in f.lower() or 'wuerstchen3' in f.lower(): - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: warn(f'Model detected as Stable Cascade model, but attempting to load using backend=original: {op}={f} size={size} MB') if devices.dtype == torch.float16: warn('Stable Cascade does not support Float16') guess = 'Stable Cascade' if 'pixart_sigma' in f.lower(): - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: warn(f'Model detected as PixArt-Sigma model, but attempting to load using backend=original: {op}={f} size={size} MB') guess = 'PixArt-Sigma' # switch for specific variant @@ -1415,7 +1416,7 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None, current_checkpoint_info = model_data.sd_refiner.sd_checkpoint_info unload_model_weights(op=op) - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: from modules import sd_hijack_inpainting sd_hijack_inpainting.do_inpainting_hijack() @@ -1472,7 +1473,7 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None, else: shared.log.debug(f'Model weights loaded: {memory_stats()}') timer.record("load") - if shared.backend == shared.Backend.ORIGINAL and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): + if not shared.native and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): lowvram.setup_for_low_vram(sd_model, shared.cmd_opts.medvram) else: move_model(sd_model, devices.device) @@ -1519,7 +1520,7 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model', current_checkpoint_info = getattr(sd_model, 'sd_checkpoint_info', None) if current_checkpoint_info is not None and checkpoint_info is not None and current_checkpoint_info.filename == checkpoint_info.filename and not force: return None - if shared.backend == shared.Backend.ORIGINAL and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): + if not shared.native and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): lowvram.send_everything_to_cpu() else: move_model(sd_model, devices.cpu) @@ -1531,12 +1532,12 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model', sd_model = None timer = Timer() # TODO implement caching after diffusers implement state_dict loading - state_dict = get_checkpoint_state_dict(checkpoint_info, timer) if shared.backend == shared.Backend.ORIGINAL else None + state_dict = get_checkpoint_state_dict(checkpoint_info, timer) if not shared.native else None checkpoint_config = sd_models_config.find_checkpoint_config(state_dict, checkpoint_info) timer.record("config") if sd_model is None or checkpoint_config != getattr(sd_model, 'used_config', None): sd_model = None - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: load_model(checkpoint_info, already_loaded_state_dict=state_dict, timer=timer, op=op) model_data.sd_dict = shared.opts.sd_model_dict else: @@ -1603,7 +1604,7 @@ def unload_model_weights(op='model'): shared.compiled_model_state.partitioned_modules.clear() if op == 'model' or op == 'dict': if model_data.sd_model: - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: from modules import sd_hijack move_model(model_data.sd_model, devices.cpu) sd_hijack.model_hijack.undo_hijack(model_data.sd_model) @@ -1615,7 +1616,7 @@ def unload_model_weights(op='model'): shared.log.debug(f'Unload weights {op}: {memory_stats()}') elif op == 'refiner': if model_data.sd_refiner: - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: from modules import sd_hijack move_model(model_data.sd_refiner, devices.cpu) sd_hijack.model_hijack.undo_hijack(model_data.sd_refiner) diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index e93998d9d..886b49ce3 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -20,7 +20,7 @@ def list_samplers(backend_name = shared.backend): global samplers # pylint: disable=global-statement global samplers_for_img2img # pylint: disable=global-statement global samplers_map # pylint: disable=global-statement - if backend_name == shared.Backend.ORIGINAL: + if not shared.native: from modules import sd_samplers_compvis, sd_samplers_kdiffusion all_samplers = [*sd_samplers_compvis.samplers_data_compvis, *sd_samplers_kdiffusion.samplers_data_k_diffusion] else: @@ -57,14 +57,14 @@ def create_sampler(name, model): if config is None or config.constructor is None: # shared.log.warning(f'Sampler: sampler="{name}" not found') return None - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: sampler = config.constructor(model) sampler.config = config sampler.name = name sampler.initialize(p=None) shared.log.debug(f'Sampler: sampler="{name}" config={config.options}') return sampler - elif shared.backend == shared.Backend.DIFFUSERS: + elif shared.native: sampler = config.constructor(model) if not hasattr(model, 'scheduler_config'): model.scheduler_config = sampler.sampler.config.copy() diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index a2b79db1e..57b4137ef 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -49,7 +49,7 @@ def single_sample_to_image(sample, approximation=None): if len(sample.shape) == 4 and sample.shape[0]: # likely animatediff latent sample = sample.permute(1, 0, 2, 3)[0] - if shared.backend == shared.Backend.DIFFUSERS: # [-x,x] to [-5,5] + if shared.native: # [-x,x] to [-5,5] sample_max = torch.max(sample) if sample_max > 5: sample = sample * (5 / sample_max) diff --git a/modules/sd_vae.py b/modules/sd_vae.py index d077a1e21..2e27393e2 100644 --- a/modules/sd_vae.py +++ b/modules/sd_vae.py @@ -54,7 +54,7 @@ def refresh_vae_list(): vae_path = shared.opts.vae_dir vae_dict.clear() vae_paths = [] - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: if sd_models.model_path is not None and os.path.isdir(sd_models.model_path): vae_paths += [ os.path.join(sd_models.model_path, 'VAE', '**/*.vae.ckpt'), @@ -73,7 +73,7 @@ def refresh_vae_list(): os.path.join(shared.opts.vae_dir, '**/*.pt'), os.path.join(shared.opts.vae_dir, '**/*.safetensors'), ] - elif shared.backend == shared.Backend.DIFFUSERS: + elif shared.native: if sd_models.model_path is not None and os.path.isdir(sd_models.model_path): vae_paths += [os.path.join(sd_models.model_path, 'VAE', '**/*.vae.safetensors')] if shared.opts.ckpt_dir is not None and os.path.isdir(shared.opts.ckpt_dir): @@ -92,7 +92,7 @@ def refresh_vae_list(): name = get_filename(filepath) if name == 'VAE': continue - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: vae_dict[name] = filepath else: if filepath.endswith(".json"): @@ -243,12 +243,12 @@ def reload_vae_weights(sd_model=None, vae_file=unspecified): vae_source = "function-argument" if loaded_vae_file == vae_file: return None - if shared.backend == shared.Backend.ORIGINAL and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): + if not shared.native and (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): lowvram.send_everything_to_cpu() # else: # sd_models.move_model(sd_model, devices.cpu) - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: sd_hijack.model_hijack.undo_hijack(sd_model) if shared.cmd_opts.rollback_vae and devices.dtype_vae == torch.bfloat16: devices.dtype_vae = torch.float16 diff --git a/modules/shared.py b/modules/shared.py index 2de3dc505..7586b3fc3 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -206,7 +206,7 @@ if cmd_opts.backend is not None: # override with args if cmd_opts.use_openvino: # override for openvino backend = Backend.DIFFUSERS from modules.intel.openvino import get_device_list as get_openvino_device_list # pylint: disable=ungrouped-imports - +native = backend == Backend.DIFFUSERS class OptionInfo: def __init__(self, default=None, label="", component=None, component_args=None, onchange=None, section=None, refresh=None, folder=None, submit=None, comment_before='', comment_after=''): @@ -340,11 +340,11 @@ def temp_disable_extensions(): for ext in disable_safe: if ext.lower() not in opts.disabled_extensions: disabled.append(ext) - if backend == Backend.DIFFUSERS: + if native: for ext in disable_diffusers: if ext.lower() not in opts.disabled_extensions: disabled.append(ext) - if backend == Backend.ORIGINAL: + if not native: for ext in disable_original: if ext.lower() not in opts.disabled_extensions: disabled.append(ext) @@ -366,13 +366,13 @@ if not (cmd_opts.lowvram or cmd_opts.medvram): if devices.backend == "directml": # Force BMM for DirectML instead of SDP - cross_attention_optimization_default = "Dynamic Attention BMM" if backend == Backend.DIFFUSERS else "Sub-quadratic" -elif backend == Backend.DIFFUSERS and (cmd_opts.lowvram or cmd_opts.medvram): + cross_attention_optimization_default = "Dynamic Attention BMM" if native else "Sub-quadratic" +elif native and (cmd_opts.lowvram or cmd_opts.medvram): cross_attention_optimization_default = "Dynamic Attention SDP" elif devices.backend == "cpu": - cross_attention_optimization_default = "Scaled-Dot-Product" if backend == Backend.DIFFUSERS else "Doggettx's" + cross_attention_optimization_default = "Scaled-Dot-Product" if native else "Doggettx's" elif devices.backend == "mps": - cross_attention_optimization_default = "Scaled-Dot-Product" if backend == Backend.DIFFUSERS else "Doggettx's" + cross_attention_optimization_default = "Scaled-Dot-Product" if native else "Doggettx's" else: # cuda, rocm, ipex cross_attention_optimization_default ="Scaled-Dot-Product" @@ -392,12 +392,12 @@ options_templates.update(options_section(('sd', "Execution & Models"), { "sd_unet": OptionInfo("None", "UNET model", gr.Dropdown, lambda: {"choices": shared_items.sd_unet_items()}, refresh=shared_items.refresh_unet_list), "sd_checkpoint_autoload": OptionInfo(True, "Model autoload on start"), "sd_model_dict": OptionInfo('None', "Use separate base dict", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints), - "stream_load": OptionInfo(False, "Load models using stream loading method", gr.Checkbox, {"visible": backend == Backend.ORIGINAL }), + "stream_load": OptionInfo(False, "Load models using stream loading method", gr.Checkbox, {"visible": not native }), "model_reuse_dict": OptionInfo(False, "Reuse loaded model dictionary", gr.Checkbox, {"visible": False}), "prompt_attention": OptionInfo("Full parser", "Prompt attention parser", gr.Radio, {"choices": ["Full parser", "Compel parser", "A1111 parser", "Fixed attention"] }), - "prompt_mean_norm": OptionInfo(True, "Prompt attention normalization", gr.Checkbox, {"visible": backend == Backend.ORIGINAL }), - "comma_padding_backtrack": OptionInfo(20, "Prompt padding", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1, "visible": backend == Backend.ORIGINAL }), - "sd_checkpoint_cache": OptionInfo(0, "Cached models", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1, "visible": backend == Backend.ORIGINAL }), + "prompt_mean_norm": OptionInfo(True, "Prompt attention normalization", gr.Checkbox, {"visible": not native }), + "comma_padding_backtrack": OptionInfo(20, "Prompt padding", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1, "visible": not native }), + "sd_checkpoint_cache": OptionInfo(0, "Cached models", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1, "visible": not native }), "sd_vae_checkpoint_cache": OptionInfo(0, "Cached VAEs", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1, "visible": False}), "sd_disable_ckpt": OptionInfo(False, "Disallow models in ckpt format", gr.Checkbox, {"visible": False}), })) @@ -416,14 +416,14 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "rollback_vae": OptionInfo(False, "Attempt VAE roll back for NaN values"), "cross_attention_sep": OptionInfo("

Attention

", "", gr.HTML), - "cross_attention_optimization": OptionInfo(cross_attention_optimization_default, "Attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention(diffusers=backend == Backend.DIFFUSERS) }), + "cross_attention_optimization": OptionInfo(cross_attention_optimization_default, "Attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention(native) }), "sdp_options": OptionInfo(sdp_options_default, "SDP options", gr.CheckboxGroup, {"choices": ['Flash attention', 'Memory attention', 'Math attention'] }), "xformers_options": OptionInfo(['Flash attention'], "xFormers options", gr.CheckboxGroup, {"choices": ['Flash attention'] }), - "dynamic_attention_slice_rate": OptionInfo(4, "Dynamic Attention slicing rate in GB", gr.Slider, {"minimum": 0.1, "maximum": 16, "step": 0.1, "visible": backend == Backend.DIFFUSERS}), - "sub_quad_sep": OptionInfo("

Sub-quadratic options

", "", gr.HTML, {"visible": backend == Backend.ORIGINAL}), - "sub_quad_q_chunk_size": OptionInfo(512, "Attention query chunk size", gr.Slider, {"minimum": 16, "maximum": 8192, "step": 8, "visible": backend == Backend.ORIGINAL}), - "sub_quad_kv_chunk_size": OptionInfo(512, "Attention kv chunk size", gr.Slider, {"minimum": 0, "maximum": 8192, "step": 8, "visible": backend == Backend.ORIGINAL}), - "sub_quad_chunk_threshold": OptionInfo(80, "Attention chunking threshold", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1, "visible": backend == Backend.ORIGINAL}), + "dynamic_attention_slice_rate": OptionInfo(4, "Dynamic Attention slicing rate in GB", gr.Slider, {"minimum": 0.1, "maximum": 16, "step": 0.1, "visible": native}), + "sub_quad_sep": OptionInfo("

Sub-quadratic options

", "", gr.HTML, {"visible": not native}), + "sub_quad_q_chunk_size": OptionInfo(512, "Attention query chunk size", gr.Slider, {"minimum": 16, "maximum": 8192, "step": 8, "visible": not native}), + "sub_quad_kv_chunk_size": OptionInfo(512, "Attention kv chunk size", gr.Slider, {"minimum": 0, "maximum": 8192, "step": 8, "visible": not native}), + "sub_quad_chunk_threshold": OptionInfo(80, "Attention chunking threshold", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1, "visible": not native}), "other_sep": OptionInfo("

Execution precision

", "", gr.HTML), "opt_channelslast": OptionInfo(False, "Use channels last "), @@ -445,7 +445,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "deep_cache_interval": OptionInfo(3, "DeepCache cache interval", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), "nncf_sep": OptionInfo("

Model Compress

", "", gr.HTML), - "nncf_compress_weights": OptionInfo([], "Compress Model weights with NNCF", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": backend == Backend.DIFFUSERS}), + "nncf_compress_weights": OptionInfo([], "Compress Model weights with NNCF", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": native}), "ipex_sep": OptionInfo("

IPEX

", "", gr.HTML, {"visible": devices.backend == "ipex"}), "ipex_optimize": OptionInfo([], "IPEX Optimize for Intel GPUs", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "Upscaler"], "visible": devices.backend == "ipex"}), @@ -668,7 +668,7 @@ options_templates.update(options_section(('ui', "User Interface Options"), { "keyedit_precision_attention": OptionInfo(0.1, "Ctrl+up/down precision when editing (attention:1.1)", gr.Slider, {"minimum": 0.01, "maximum": 0.2, "step": 0.001, "visible": False}), "keyedit_precision_extra": OptionInfo(0.05, "Ctrl+up/down precision when editing ", gr.Slider, {"minimum": 0.01, "maximum": 0.2, "step": 0.001, "visible": False}), "keyedit_delimiters": OptionInfo(r".,\/!?%^*;:{}=`~()", "Ctrl+up/down word delimiters", gr.Textbox, { "visible": False }), - "quicksettings_list": OptionInfo(["sd_model_checkpoint"] if backend == Backend.ORIGINAL else ["sd_model_checkpoint", "sd_model_refiner"], "Quicksettings list", gr.Dropdown, lambda: {"multiselect":True, "choices": list(opts.data_labels.keys())}), + "quicksettings_list": OptionInfo(["sd_model_checkpoint"], "Quicksettings list", gr.Dropdown, lambda: {"multiselect":True, "choices": list(opts.data_labels.keys())}), "ui_scripts_reorder": OptionInfo("", "UI scripts order", gr.Textbox, { "visible": False }), })) @@ -815,7 +815,7 @@ options_templates.update(options_section(('extra_networks', "Extra Networks"), { "extra_network_reference": OptionInfo(False, "Use reference values when available", gr.Checkbox), "extra_network_skip_indexing": OptionInfo(False, "Build info on first access", gr.Checkbox), "extra_networks_default_multiplier": OptionInfo(1.0, "Default multiplier for extra networks", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - "diffusers_convert_embed": OptionInfo(False, "Auto-convert SD 1.5 embeddings to SDXL ", gr.Checkbox, {"visible": backend==Backend.DIFFUSERS}), + "diffusers_convert_embed": OptionInfo(False, "Auto-convert SD 1.5 embeddings to SDXL ", gr.Checkbox, {"visible": native}), "extra_networks_sep3": OptionInfo("

Extra networks settings

", "", gr.HTML), "extra_networks_styles": OptionInfo(True, "Show built-in styles"), "lora_preferred_name": OptionInfo("filename", "LoRA preferred name", gr.Radio, {"choices": ["filename", "alias"]}), diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index bf3e19a44..e0fcbc55f 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -27,7 +27,7 @@ def list_textual_inversion_templates(): def list_embeddings(*dirs): - is_ext = extension_filter(['.SAFETENSORS', '.PT' ] + ( ['.PNG', '.WEBP', '.JXL', '.AVIF', '.BIN' ] if shared.backend != shared.Backend.DIFFUSERS else [] )) + is_ext = extension_filter(['.SAFETENSORS', '.PT' ] + ( ['.PNG', '.WEBP', '.JXL', '.AVIF', '.BIN' ] if not shared.native else [] )) is_not_preview = lambda fp: not next(iter(os.path.splitext(fp))).upper().endswith('.PREVIEW') # pylint: disable=unnecessary-lambda-assignment return list(filter(lambda fp: is_ext(fp) and is_not_preview(fp) and os.stat(fp).st_size > 0, directory_files(*dirs))) @@ -138,7 +138,7 @@ class EmbeddingDatabase: return embedding def get_expected_shape(self): - if shared.backend == shared.Backend.DIFFUSERS: + if shared.native: return 0 if not shared.sd_loaded: shared.log.error('Model not loaded') @@ -302,7 +302,7 @@ class EmbeddingDatabase: else: raise RuntimeError(f"Couldn't identify {filename} as textual inversion embedding") - if shared.backend == shared.Backend.DIFFUSERS: + if shared.native: return emb vec = emb.detach().to(devices.device, dtype=torch.float32) @@ -326,7 +326,7 @@ class EmbeddingDatabase: if not os.path.isdir(embdir.path): return file_paths = list_embeddings(embdir.path) - if shared.backend == shared.Backend.DIFFUSERS: + if shared.native: self.load_diffusers_embedding(file_paths) else: for file_path in file_paths: diff --git a/modules/ui.py b/modules/ui.py index 3d9041f78..70128e0f9 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -139,7 +139,7 @@ def create_ui(startup_timer = None): modules.scripts.scripts_current = None with gr.Blocks(analytics_enabled=False) as control_interface: - if shared.backend == shared.Backend.DIFFUSERS: + if shared.native: from modules import ui_control ui_control.create_ui() timer.startup.record("ui-control") diff --git a/modules/ui_common.py b/modules/ui_common.py index 9ad1e14c1..1b8c5aade 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -262,7 +262,7 @@ def create_output_panel(tabname, preview=True, prompt=None, height=None): clip_files.click(fn=None, _js='clip_gallery_urls', inputs=[result_gallery], outputs=[]) save = gr.Button('Save', elem_id=f'save_{tabname}') delete = gr.Button('Delete', elem_id=f'delete_{tabname}') - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: buttons = generation_parameters_copypaste.create_buttons(["img2img", "inpaint", "extras"]) else: buttons = generation_parameters_copypaste.create_buttons(["txt2img", "img2img", "control", "extras"]) @@ -389,7 +389,7 @@ def update_token_counter(text, steps): return f"{token_count}/{max_length}" from modules import extra_networks prompt, _ = extra_networks.parse_prompt(text) - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: from modules import sd_hijack try: _, prompt_flat_list, _ = prompt_parser.get_multicond_prompt_list([text]) @@ -399,7 +399,7 @@ def update_token_counter(text, steps): flat_prompts = reduce(lambda list1, list2: list1+list2, prompt_schedules) prompts = [prompt_text for _step, prompt_text in flat_prompts] token_count, max_length = max([sd_hijack.model_hijack.get_prompt_lengths(prompt) for prompt in prompts], key=lambda args: args[0]) - elif shared.backend == shared.Backend.DIFFUSERS: + elif shared.native: if shared.sd_loaded and hasattr(shared.sd_model, 'tokenizer') and shared.sd_model.tokenizer is not None: has_bos_token = shared.sd_model.tokenizer.bos_token_id is not None has_eos_token = shared.sd_model.tokenizer.eos_token_id is not None diff --git a/modules/ui_control.py b/modules/ui_control.py index f8e805e33..1bf378656 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -67,7 +67,7 @@ def generate_click(job_id: str, active_tab: str, *args): def create_ui(_blocks: gr.Blocks=None): helpers.initialize() - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: with gr.Blocks(analytics_enabled = False) as control_ui: pass return [(control_ui, 'Control', 'control')] diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 8f93aef24..7a49ce660 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -224,7 +224,7 @@ class ExtraNetworksPage: tgt = tgt.path if os.path.join(paths.models_path, 'Reference') in tgt: subdirs['Reference'] = 1 - if shared.backend == shared.Backend.DIFFUSERS and shared.opts.diffusers_dir in tgt: + if shared.native and shared.opts.diffusers_dir in tgt: subdirs[os.path.basename(shared.opts.diffusers_dir)] = 1 if 'models--' in tgt: continue diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py index fb75de1f2..e1e5db820 100644 --- a/modules/ui_extra_networks_checkpoints.py +++ b/modules/ui_extra_networks_checkpoints.py @@ -16,7 +16,7 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): def list_reference(self): # pylint: disable=inconsistent-return-statements for k, v in shared.reference_models.items(): - if shared.backend != shared.Backend.DIFFUSERS: + if not shared.native: if not v.get('original', False): continue url = v.get('alt', None) or v['path'] @@ -82,7 +82,7 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): return items def allowed_directories_for_previews(self): - if shared.backend == shared.Backend.DIFFUSERS: + if shared.native: return [v for v in [shared.opts.ckpt_dir, shared.opts.diffusers_dir, reference_dir] if v is not None] else: return [v for v in [shared.opts.ckpt_dir, reference_dir, sd_models.model_path] if v is not None] diff --git a/modules/ui_extra_networks_textual_inversion.py b/modules/ui_extra_networks_textual_inversion.py index 1859274da..3b0ec0948 100644 --- a/modules/ui_extra_networks_textual_inversion.py +++ b/modules/ui_extra_networks_textual_inversion.py @@ -13,7 +13,7 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage): def refresh(self): if sd_models.model_data.sd_model is None: return - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings(force_reload=True) elif hasattr(sd_models.model_data.sd_model, 'embedding_db'): sd_models.model_data.sd_model.embedding_db.load_textual_inversion_embeddings(force_reload=True) @@ -48,7 +48,7 @@ class ExtraNetworksPageTextualInversion(ui_extra_networks.ExtraNetworksPage): for embedding_path in candidates ] - elif shared.backend == shared.Backend.ORIGINAL: + elif not shared.native: self.embeddings = list(sd_hijack.model_hijack.embedding_db.word_embeddings.values()) elif hasattr(sd_models.model_data.sd_model, 'embedding_db'): self.embeddings = list(sd_models.model_data.sd_model.embedding_db.word_embeddings.values()) diff --git a/modules/ui_img2img.py b/modules/ui_img2img.py index 09d0c5022..e4faa0abc 100644 --- a/modules/ui_img2img.py +++ b/modules/ui_img2img.py @@ -141,7 +141,7 @@ def create_ui(): with gr.Row(): inpainting_mask_invert = gr.Radio(label='Mode', choices=['masked', 'invert'], value='masked', type="index", elem_id="img2img_mask_mode") inpaint_full_res = gr.Radio(label="Inpaint area", choices=["full", "masked"], type="index", value="full", elem_id="img2img_inpaint_full_res") - inpainting_fill = gr.Radio(label='Masked content', choices=['fill', 'original', 'noise', 'nothing'], value='original', type="index", elem_id="img2img_inpainting_fill", visible=shared.backend == shared.Backend.ORIGINAL) + inpainting_fill = gr.Radio(label='Masked content', choices=['fill', 'original', 'noise', 'nothing'], value='original', type="index", elem_id="img2img_inpainting_fill", visible=not shared.native) def select_img2img_tab(tab): return gr.update(visible=tab in [2, 3, 4]), gr.update(visible=tab == 3) diff --git a/modules/ui_sections.py b/modules/ui_sections.py index b6082e839..c2bd4ecd9 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -166,18 +166,18 @@ def create_advanced_inputs(tab, base=True): cfg_scale, cfg_end = None, None with gr.Row(): image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Secondary guidance', value=6.0, elem_id=f"{tab}_image_cfg_scale") - diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Rescale guidance', value=0.7, elem_id=f"{tab}_image_cfg_rescale", visible=shared.backend == shared.Backend.DIFFUSERS) + diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Rescale guidance', value=0.7, elem_id=f"{tab}_image_cfg_rescale", visible=shared.native) with gr.Row(): - diffusers_pag_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.05, label='Attention guidance', value=0.0, elem_id=f"{tab}_pag_scale", visible=shared.backend == shared.Backend.DIFFUSERS) - diffusers_pag_adaptive = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Adaptive scaling', value=0.5, elem_id=f"{tab}_pag_adaptive", visible=shared.backend == shared.Backend.DIFFUSERS) + diffusers_pag_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.05, label='Attention guidance', value=0.0, elem_id=f"{tab}_pag_scale", visible=shared.native) + diffusers_pag_adaptive = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Adaptive scaling', value=0.5, elem_id=f"{tab}_pag_adaptive", visible=shared.native) with gr.Row(): clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=0, maximum=12, step=0.1, elem_id=f"{tab}_clip_skip", interactive=True) return cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, diffusers_pag_scale, diffusers_pag_adaptive, cfg_end def create_correction_inputs(tab): - with gr.Accordion(open=False, label="Corrections", elem_id=f"{tab}_corrections", elem_classes=["small-accordion"], visible=shared.backend == shared.Backend.DIFFUSERS): - with gr.Group(visible=shared.backend == shared.Backend.DIFFUSERS): + with gr.Accordion(open=False, label="Corrections", elem_id=f"{tab}_corrections", elem_classes=["small-accordion"], visible=shared.native): + with gr.Group(visible=shared.native): with gr.Row(elem_id=f"{tab}_hdr_mode_row"): hdr_mode = gr.Dropdown(label="Mode", choices=["Relative values", "Absolute values"], type="index", value="Relative values", elem_id=f"{tab}_hdr_mode", show_label=False) gr.HTML('
') @@ -243,7 +243,7 @@ def create_sampler_options(tabname): return '999,845,730,587,443,310,193,116,53,13' return '' - if shared.backend == shared.Backend.ORIGINAL: + if not shared.native: with gr.Row(elem_classes=['flex-break']): options = ['brownian noise', 'discard penultimate sigma'] values = [] @@ -292,7 +292,7 @@ def create_hires_inputs(tab): with gr.Row(elem_id=f"{tab}_hires_row2"): hr_second_pass_steps = gr.Slider(minimum=0, maximum=99, step=1, label='HiRes steps', elem_id=f"{tab}_steps_alt", value=20) denoising_strength = gr.Slider(minimum=0.0, maximum=0.99, step=0.01, label='Strength', value=0.3, elem_id=f"{tab}_denoising_strength") - with gr.Group(visible=shared.backend == shared.Backend.DIFFUSERS): + with gr.Group(visible=shared.native): with gr.Row(elem_id=f"{tab}_refiner_row1", variant="compact"): refiner_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Refiner start', value=0.0, elem_id=f"{tab}_refiner_start") refiner_steps = gr.Slider(minimum=0, maximum=99, step=1, label="Refiner steps", elem_id=f"{tab}_refiner_steps", value=10) diff --git a/scripts/animatediff.py b/scripts/animatediff.py index c1df67175..8c82a2cfc 100644 --- a/scripts/animatediff.py +++ b/scripts/animatediff.py @@ -50,7 +50,7 @@ orig_pipe = None # original sd_model pipeline def set_adapter(adapter_name: str = 'None'): if not shared.sd_loaded: return - if shared.backend != shared.Backend.DIFFUSERS: + if not shared.native: shared.log.warning('AnimateDiff: not in diffusers mode') return global motion_adapter, loaded_adapter, orig_pipe # pylint: disable=global-statement @@ -135,7 +135,7 @@ class Script(scripts.Script): return 'AnimateDiff' def show(self, _is_img2img): - return scripts.AlwaysVisible if shared.backend == shared.Backend.DIFFUSERS else False + return scripts.AlwaysVisible if shared.native else False def ui(self, _is_img2img): diff --git a/scripts/blipdiffusion.py b/scripts/blipdiffusion.py index f2005a0b3..39d8974e1 100644 --- a/scripts/blipdiffusion.py +++ b/scripts/blipdiffusion.py @@ -10,7 +10,7 @@ class Script(scripts.Script): return title def show(self, is_img2img): - return is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + return is_img2img if shared.native else False def ui(self, _is_img2img): with gr.Row(): diff --git a/scripts/demofusion.py b/scripts/demofusion.py index eecae05a3..95e58a74e 100644 --- a/scripts/demofusion.py +++ b/scripts/demofusion.py @@ -1225,7 +1225,7 @@ class Script(scripts.Script): return 'DemoFusion' def show(self, is_img2img): - return not is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + return not is_img2img if shared.native else False # return signature is array of gradio components def ui(self, _is_img2img): diff --git a/scripts/differential_diffusion.py b/scripts/differential_diffusion.py index 4ab1bcaa9..b48e0f6e6 100644 --- a/scripts/differential_diffusion.py +++ b/scripts/differential_diffusion.py @@ -1875,7 +1875,7 @@ class Script(scripts.Script): return 'Differential diffusion' def show(self, is_img2img): - return is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + return is_img2img if shared.native else False def ui(self, _is_img2img): with gr.Row(): diff --git a/scripts/example.py b/scripts/example.py index a3f52ea7b..aba1fba5c 100644 --- a/scripts/example.py +++ b/scripts/example.py @@ -67,7 +67,7 @@ class Script(scripts.Script): return title def show(self, is_img2img): - if shared.backend == shared.Backend.DIFFUSERS: + if shared.native: return img2img if is_img2img else txt2img return False diff --git a/scripts/image2video.py b/scripts/image2video.py index 0c8c476a2..332972a6d 100644 --- a/scripts/image2video.py +++ b/scripts/image2video.py @@ -16,7 +16,7 @@ class Script(scripts.Script): return 'Image-to-Video' def show(self, is_img2img): - return is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + return is_img2img if shared.native else False # return False # return signature is array of gradio components diff --git a/scripts/init_latents.py b/scripts/init_latents.py index d689e07f7..a21c11f21 100644 --- a/scripts/init_latents.py +++ b/scripts/init_latents.py @@ -8,7 +8,7 @@ class Script(scripts.Script): return 'Init Latents' def show(self, is_img2img): - return scripts.AlwaysVisible if shared.backend == shared.Backend.DIFFUSERS else False + return scripts.AlwaysVisible if shared.native else False @staticmethod def get_latents(p): @@ -31,7 +31,7 @@ class Script(scripts.Script): def process_batch(self, p: processing.StableDiffusionProcessing, *args, **kwargs): # pylint: disable=arguments-differ from modules.processing_helpers import create_random_tensors - if shared.backend != shared.Backend.DIFFUSERS: + if not shared.native: return args = list(args) if p.subseed_strength != 0 and getattr(shared.sd_model, '_execution_device', None) is not None: diff --git a/scripts/ipadapter.py b/scripts/ipadapter.py index af1e14c35..dab1e0fba 100644 --- a/scripts/ipadapter.py +++ b/scripts/ipadapter.py @@ -14,7 +14,7 @@ class Script(scripts.Script): return 'IP Adapters' def show(self, is_img2img): - return scripts.AlwaysVisible if shared.backend == shared.Backend.DIFFUSERS else False + return scripts.AlwaysVisible if shared.native else False def load_images(self, files): init_images = [] @@ -83,7 +83,7 @@ class Script(scripts.Script): return [num_adapters] + adapters + scales + files + starts + ends + masks + [layers_active] + [layers] def process(self, p: processing.StableDiffusionProcessing, *args): # pylint: disable=arguments-differ - if shared.backend != shared.Backend.DIFFUSERS: + if not shared.native: return args = list(args) if args is not None else [] if len(args) == 0: diff --git a/scripts/kohya_hires_fix.py b/scripts/kohya_hires_fix.py index 2a50968af..090edcaf9 100644 --- a/scripts/kohya_hires_fix.py +++ b/scripts/kohya_hires_fix.py @@ -8,7 +8,7 @@ class Script(scripts.Script): return 'Kohya HiRes Fix' def show(self, is_img2img): - return not is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + return not is_img2img if shared.native else False # return signature is array of gradio components def ui(self, _is_img2img): diff --git a/scripts/layerdiffuse.py b/scripts/layerdiffuse.py index 9c2de4ade..a1e15aa8b 100644 --- a/scripts/layerdiffuse.py +++ b/scripts/layerdiffuse.py @@ -8,7 +8,7 @@ class Script(scripts.Script): return 'LayerDiffuse' def show(self, is_img2img): - return True if shared.backend == shared.Backend.DIFFUSERS else False + return True if shared.native else False def apply(self): from modules import layerdiffuse diff --git a/scripts/ledits.py b/scripts/ledits.py index 71ed2c300..1a0e929f0 100644 --- a/scripts/ledits.py +++ b/scripts/ledits.py @@ -8,7 +8,7 @@ class Script(scripts.Script): return 'LEdits++' def show(self, is_img2img): - return is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + return is_img2img if shared.native else False # return signature is array of gradio components def ui(self, _is_img2img): diff --git a/scripts/mixture_tiling.py b/scripts/mixture_tiling.py index 4425725bf..5dcaf0156 100644 --- a/scripts/mixture_tiling.py +++ b/scripts/mixture_tiling.py @@ -29,7 +29,7 @@ class Script(scripts.Script): return 'Mixture tiling' def show(self, is_img2img): - return not is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + return not is_img2img if shared.native else False def ui(self, _is_img2img): with gr.Row(): diff --git a/scripts/mulan.py b/scripts/mulan.py index 3b5baa570..aa5311f64 100644 --- a/scripts/mulan.py +++ b/scripts/mulan.py @@ -50,7 +50,7 @@ class Script(scripts.Script): def show(self, is_img2img): if shared.cmd_opts.experimental: - return True if shared.backend == shared.Backend.DIFFUSERS else False + return True if shared.native else False else: return False diff --git a/scripts/regional_prompting.py b/scripts/regional_prompting.py index e18d85600..ab1f4a902 100644 --- a/scripts/regional_prompting.py +++ b/scripts/regional_prompting.py @@ -24,7 +24,7 @@ class Script(scripts.Script): return 'Regional prompting' def show(self, is_img2img): - return not is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + return not is_img2img if shared.native else False def change(self, mode): return [gr.update(visible='Col' in mode or 'Row' in mode), gr.update(visible='Prompt' in mode)] diff --git a/scripts/stablevideodiffusion.py b/scripts/stablevideodiffusion.py index 41b588eb4..585871edc 100644 --- a/scripts/stablevideodiffusion.py +++ b/scripts/stablevideodiffusion.py @@ -19,7 +19,7 @@ class Script(scripts.Script): return 'Stable Video Diffusion' def show(self, is_img2img): - return is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + return is_img2img if shared.native else False # return signature is array of gradio components def ui(self, _is_img2img): diff --git a/scripts/t_gate.py b/scripts/t_gate.py index 7c55f998d..3bd51445d 100644 --- a/scripts/t_gate.py +++ b/scripts/t_gate.py @@ -8,7 +8,7 @@ class Script(scripts.Script): return 'T-Gate' def show(self, is_img2img): - return not is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + return not is_img2img if shared.native else False # return signature is array of gradio components def ui(self, _is_img2img): diff --git a/scripts/text2video.py b/scripts/text2video.py index ada78e849..2c93abf27 100644 --- a/scripts/text2video.py +++ b/scripts/text2video.py @@ -26,7 +26,7 @@ class Script(scripts.Script): return 'Text-to-Video' def show(self, is_img2img): - return not is_img2img if shared.backend == shared.Backend.DIFFUSERS else False + return not is_img2img if shared.native else False # return signature is array of gradio components def ui(self, _is_img2img): diff --git a/scripts/x_adapter.py b/scripts/x_adapter.py index f22facb4c..58d6fb9eb 100644 --- a/scripts/x_adapter.py +++ b/scripts/x_adapter.py @@ -16,7 +16,7 @@ class Script(scripts.Script): def show(self, is_img2img): return False - # return True if shared.backend == shared.Backend.DIFFUSERS else False + # return True if shared.native else False def ui(self, _is_img2img): with gr.Row(): From db9718eee6cfd3fa00155c1d6798a8999e0aa06c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 7 Jun 2024 09:26:51 -0400 Subject: [PATCH 22/34] add torch full deterministic mode --- CHANGELOG.md | 3 ++ installer.py | 10 +++--- modules/devices.py | 10 ++++-- modules/hidiffusion/__init__.py | 6 ++-- modules/hidiffusion/hidiffusion.py | 58 ++++++++++++++---------------- modules/pag/__init__.py | 10 +++--- modules/processing_diffusers.py | 4 +-- modules/processing_helpers.py | 3 ++ modules/shared.py | 8 +++-- 9 files changed, 61 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bdcac483..5a698338a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,9 @@ *note*: alternative to regular hidiffusion method, but with different approach to scaling - additional built-in 4 great custom trained **ControlNet SDXL** models from Xinsir: OpenPose, Canny, Scribble, AnimePainter thanks @lbeltrame +- add torch **full deterministic mode** + enable in settings -> compute -> use deterministic mode + typical differences are not large and its disabled by default as it does have some performance impact - lower overhead on generate calls - cumulative fixes since the last release - add python version check for torch-directml diff --git a/installer.py b/installer.py index 532e62b25..f965c76d9 100644 --- a/installer.py +++ b/installer.py @@ -991,19 +991,19 @@ def check_ui(ver): return if ver['branch'] == ver['ui']: return - log.warning(f'Branch mismatch: sdnext={ver["branch"]} ui={ver["ui"]}') + log.debug(f'Branch mismatch: sdnext={ver["branch"]} ui={ver["ui"]}') cwd = os.getcwd() try: os.chdir('extensions-builtin/sdnext-modernui') - git('checkout ' + ver['branch']) + git('checkout ' + ver['branch'], ignore=True) os.chdir(cwd) ver = get_version(force=True) if ver['branch'] == ver['ui']: - log.info(f'Branch synchronized: {ver["branch"]}') + log.debug(f'Branch synchronized: {ver["branch"]}') else: - log.error(f'Branch synchronize: sdnext={ver["branch"]} ui={ver["ui"]}') + log.debug(f'Branch synch failed: sdnext={ver["branch"]} ui={ver["ui"]}') except Exception as e: - log.error(f'Branch switch: {e}') + log.debug(f'Branch switch: {e}') os.chdir(cwd) diff --git a/modules/devices.py b/modules/devices.py index 36425ff39..8ff3ad04b 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -232,9 +232,13 @@ def set_cuda_params(): if torch.backends.cudnn.is_available(): try: torch.backends.cudnn.deterministic = shared.opts.cudnn_deterministic + torch.use_deterministic_algorithms(shared.opts.cudnn_deterministic) + log.debug(f'Torch mode: deterministic={shared.opts.cudnn_deterministic}') + if shared.opts.cudnn_deterministic: + os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8' torch.backends.cudnn.benchmark = True if shared.opts.cudnn_benchmark: - log.debug('Torch enable cuDNN benchmark') + log.debug('Torch cuDNN: enable benchmark') torch.backends.cudnn.benchmark_limit = 0 torch.backends.cudnn.allow_tf32 = True except Exception: @@ -363,10 +367,12 @@ def cond_cast_float(tensor): return tensor.float() if unet_needs_upcast else tensor -def randn(seed, shape): +def randn(seed, shape=None): torch.manual_seed(seed) if backend == 'ipex': torch.xpu.manual_seed_all(seed) + if shape is None: + return None if device.type == 'mps': return torch.randn(shape, device=cpu).to(device) elif shared.opts.diffusers_generator_device == "CPU": diff --git a/modules/hidiffusion/__init__.py b/modules/hidiffusion/__init__.py index 2ef4c032a..7ca7e253b 100644 --- a/modules/hidiffusion/__init__.py +++ b/modules/hidiffusion/__init__.py @@ -5,13 +5,13 @@ from modules import shared from modules.hidiffusion import hidiffusion -def apply_hidiffusion(p, model_type): +def apply(p, model_type): if not shared.native: return if model_type not in ['sd', 'sdxl'] and p.hidiffusion: shared.log.warning(f'HiDiffusion: class={shared.sd_model.__class__.__name__} not supported') return - remove_hidiffusion(p) + unapply() if getattr(p, 'hidiffusion', False) is True: t0 = time.time() hidiffusion.is_aggressive_raunet = shared.opts.hidiffusion_steps > 0 @@ -38,6 +38,6 @@ def apply_hidiffusion(p, model_type): shared.log.debug(f'HiDiffusion apply: raunet={shared.opts.hidiffusion_raunet} attn={shared.opts.hidiffusion_attn} aggressive={shared.opts.hidiffusion_steps > 0}:{shared.opts.hidiffusion_steps} t1={shared.opts.hidiffusion_t1} t2={shared.opts.hidiffusion_t2} time={t1-t0:.2f} type={shared.sd_model_type} width={p.width} height={p.height}') -def remove_hidiffusion(p): +def unapply(): if hasattr(shared.sd_model, "unet"): hidiffusion.remove_hidiffusion(shared.sd_model) diff --git a/modules/hidiffusion/hidiffusion.py b/modules/hidiffusion/hidiffusion.py index e24bce020..dc0f8d991 100644 --- a/modules/hidiffusion/hidiffusion.py +++ b/modules/hidiffusion/hidiffusion.py @@ -3,7 +3,6 @@ import torch import torch.nn.functional as F from diffusers.utils.torch_utils import is_torch_version from diffusers.pipelines import auto_pipeline -from modules.shared import log def sd15_hidiffusion_key(): @@ -229,7 +228,7 @@ def make_diffusers_transformer_block(block_class: Type[torch.nn.Module]) -> Type norm_hidden_states = self.norm2(hidden_states) norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp if self._chunk_size is not None: - ff_output = _chunked_feed_forward(self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size) + ff_output = _chunked_feed_forward(self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size) # TODO hidiffusion undefined else: ff_output = self.ff(norm_hidden_states) if self.use_ada_layer_norm_zero: @@ -268,7 +267,7 @@ def make_diffusers_cross_attn_down_block(block_class: Type[torch.nn.Module]) -> encoder_attention_mask: Optional[torch.FloatTensor] = None, additional_residuals: Optional[torch.FloatTensor] = None, ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: - self.max_timestep = self.info['pipeline']._num_timesteps + self.max_timestep = self.info['pipeline']._num_timesteps # pylint: disable=protected-access # self.max_timestep = len(self.info['scheduler'].timesteps) ori_H, ori_W = self.info['size'] if self.model == 'sd15': @@ -303,7 +302,7 @@ def make_diffusers_cross_attn_down_block(block_class: Type[torch.nn.Module]) -> self.T1 = int(self.max_timestep * self.T1_ratio) output_states = () - cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 + _scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 # TODO hidiffusion unused blocks = list(zip(self.resnets, self.attentions)) @@ -407,7 +406,7 @@ def make_diffusers_cross_attn_up_block(block_class: Type[torch.nn.Module]) -> Ty return F.interpolate(first, scale_factor=rescale, mode='bicubic') return first - self.max_timestep = self.info['pipeline']._num_timesteps + self.max_timestep = self.info['pipeline']._num_timesteps # pylint: disable=protected-access ori_H, ori_W = self.info['size'] if self.model == 'sd15': if ori_H < 256 or ori_W < 256: @@ -489,8 +488,8 @@ def make_diffusers_downsampler_block(block_class: Type[torch.nn.Module]) -> Type aggressive_raunet = False max_timestep = 50 - def forward(self, hidden_states: torch.Tensor, scale = 1.0) -> torch.Tensor: - self.max_timestep = self.info['pipeline']._num_timesteps + def forward(self, hidden_states: torch.Tensor, scale = 1.0) -> torch.Tensor: # pylint: disable=unused-argument + self.max_timestep = self.info['pipeline']._num_timesteps # pylint: disable=protected-access # self.max_timestep = len(self.info['scheduler'].timesteps) ori_H, ori_W = self.info['size'] if self.model == 'sd15': @@ -522,20 +521,20 @@ def make_diffusers_downsampler_block(block_class: Type[torch.nn.Module]) -> Type else: self.T1 = int(self.max_timestep * self.T1_ratio) if self.timestep < self.T1: - self.ori_stride = self.stride - self.ori_padding = self.padding - self.ori_dilation = self.dilation - self.stride = (4,4) - self.padding = (2,2) - self.dilation = (2,2) + self.ori_stride = self.stride # pylint: disable=access-member-before-definition, attribute-defined-outside-init + self.ori_padding = self.padding # pylint: disable=access-member-before-definition, attribute-defined-outside-init + self.ori_dilation = self.dilation # pylint: disable=access-member-before-definition, attribute-defined-outside-init + self.stride = (4,4) # pylint: disable=access-member-before-definition, attribute-defined-outside-init + self.padding = (2,2) # pylint: disable=access-member-before-definition, attribute-defined-outside-init + self.dilation = (2,2) # pylint: disable=access-member-before-definition, attribute-defined-outside-init hidden_states = F.conv2d( hidden_states, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups ) if self.timestep < self.T1: - self.stride = self.ori_stride - self.padding = self.ori_padding - self.dilation = self.ori_dilation + self.stride = self.ori_stride # pylint: disable=access-member-before-definition, attribute-defined-outside-init + self.padding = self.ori_padding # pylint: disable=access-member-before-definition, attribute-defined-outside-init + self.dilation = self.ori_dilation # pylint: disable=access-member-before-definition, attribute-defined-outside-init self.timestep += 1 if self.timestep == self.max_timestep: self.timestep = 0 @@ -557,8 +556,8 @@ def make_diffusers_upsampler_block(block_class: Type[torch.nn.Module]) -> Type[t aggressive_raunet = False max_timestep = 50 - def forward(self, hidden_states: torch.Tensor, scale = 1.0) -> torch.Tensor: - self.max_timestep = self.info['pipeline']._num_timesteps + def forward(self, hidden_states: torch.Tensor, scale = 1.0) -> torch.Tensor: # pylint: disable=unused-argument + self.max_timestep = self.info['pipeline']._num_timesteps # pylint: disable=protected-access # self.max_timestep = len(self.info['scheduler'].timesteps) ori_H, ori_W = self.info['size'] if self.model == 'sd15': @@ -645,24 +644,21 @@ def apply_hidiffusion( modified_key = sd15_hidiffusion_key() for key, module in diffusion_model.named_modules(): if apply_raunet and key in modified_key['down_module_key']: - make_block_fn = make_diffusers_downsampler_block - module.__class__ = make_block_fn(module.__class__) + module.__class__ = make_diffusers_downsampler_block(module.__class__) module.switching_threshold_ratio = 'T1_ratio' if apply_raunet and key in modified_key['down_module_key_extra']: - make_block_fn = make_diffusers_cross_attn_down_block - module.__class__ = make_block_fn(module.__class__) + module.__class__ = make_diffusers_cross_attn_down_block(module.__class__) module.switching_threshold_ratio = 'T2_ratio' if apply_raunet and key in modified_key['up_module_key']: - make_block_fn = make_diffusers_upsampler_block - module.__class__ = make_block_fn(module.__class__) + module.__class__ = make_diffusers_upsampler_block(module.__class__) module.switching_threshold_ratio = 'T1_ratio' if apply_raunet and key in modified_key['up_module_key_extra']: - make_block_fn = make_diffusers_cross_attn_up_block - module.__class__ = make_block_fn(module.__class__) + module.__class__ = make_diffusers_cross_attn_up_block(module.__class__) module.switching_threshold_ratio = 'T2_ratio' if apply_window_attn and key in modified_key['windown_attn_module_key']: - make_block_fn = make_diffusers_transformer_block - module.__class__ = make_block_fn(module.__class__) + module.__class__ = make_diffusers_transformer_block(module.__class__) + if hasattr(module, "_patched_forward"): + module.forward = module._patched_forward # pylint: disable=protected-access module.model = 'sd15' module.info = diffusion_model.info @@ -685,7 +681,7 @@ def apply_hidiffusion( if apply_window_attn and key in modified_key['windown_attn_module_key']: module.__class__ = make_diffusers_transformer_block(module.__class__) if hasattr(module, "_patched_forward"): - module.forward = module._patched_forward + module.forward = module._patched_forward # pylint: disable=protected-access module.model = 'sdxl' module.info = diffusion_model.info else: @@ -702,7 +698,7 @@ def remove_hidiffusion(model: torch.nn.Module): module.info["hooks"].clear() del module.info if hasattr(module, "_forward"): - module.forward = module._forward + module.forward = module._forward # pylint: disable=protected-access if hasattr(module, "_parent"): - module.__class__ = module._parent + module.__class__ = module._parent # pylint: disable=protected-access return model diff --git a/modules/pag/__init__.py b/modules/pag/__init__.py index 8e43bb30b..484fccc13 100644 --- a/modules/pag/__init__.py +++ b/modules/pag/__init__.py @@ -10,15 +10,15 @@ orig_pipeline = None def apply(p: processing.StableDiffusionProcessing): # pylint: disable=arguments-differ global orig_pipeline # pylint: disable=global-statement - c = shared.sd_model.__class__ if shared.sd_loaded else None if not shared.native: return None - if p.pag_scale == 0: + c = shared.sd_model.__class__ if shared.sd_loaded else None + if c == StableDiffusionPAGPipeline or c == StableDiffusionXLPAGPipeline: unapply() return None - if c == StableDiffusionPAGPipeline or c == StableDiffusionXLPAGPipeline: - pass - elif detect.is_sd15(c): + if p.pag_scale == 0: + return + if detect.is_sd15(c): orig_pipeline = shared.sd_model shared.sd_model = sd_models.switch_pipe(StableDiffusionPAGPipeline, shared.sd_model) elif detect.is_sdxl(c): diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 8b8326aca..55f1f9af9 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -113,7 +113,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): t0 = time.time() sd_models_compile.check_deepcache(enable=True) sd_models.move_model(shared.sd_model, devices.device) - hidiffusion.apply_hidiffusion(p, shared.sd_model_type) + hidiffusion.apply(p, shared.sd_model_type) # if 'image' in base_args: # base_args['image'] = set_latents(p) if hasattr(shared.sd_model, 'tgate') and getattr(p, 'gate_step', -1) > 0: @@ -123,7 +123,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): output = shared.sd_model(**base_args) if isinstance(output, dict): output = SimpleNamespace(**output) - hidiffusion.remove_hidiffusion(p) + hidiffusion.unapply() sd_models_compile.openvino_post_compile(op="base") # only executes on compiled vino models sd_models_compile.check_deepcache(enable=False) if shared.cmd_opts.profile: diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index 30199dd98..be04ea8d4 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -482,7 +482,10 @@ def get_generator(p): else: generator_device = devices.cpu if shared.opts.diffusers_generator_device == "CPU" else shared.device try: + devices.randn(p.seeds[0]) generator = [torch.Generator(generator_device).manual_seed(s) for s in p.seeds] + seeds = [g.initial_seed() for g in generator] + shared.log.debug(f'Torch generator: device={generator_device} seeds={seeds}') except Exception as e: shared.log.error(f'Torch generator: seeds={p.seeds} device={generator_device} {e}') generator = None diff --git a/modules/shared.py b/modules/shared.py index 7586b3fc3..f7395ce47 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -406,6 +406,9 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "math_sep": OptionInfo("

Execution precision

", "", gr.HTML), "precision": OptionInfo("Autocast", "Precision type", gr.Radio, {"choices": ["Autocast", "Full"]}), "cuda_dtype": OptionInfo("FP32" if sys.platform == "darwin" or cmd_opts.use_openvino else "BF16" if devices.backend == "ipex" else "FP16", "Device precision type", gr.Radio, {"choices": ["FP32", "FP16", "BF16"]}), + "cudnn_deterministic": OptionInfo(False, "Use deterministic mode"), + + "model_sep": OptionInfo("

Model options

", "", gr.HTML), "no_half": OptionInfo(False if not cmd_opts.use_openvino else True, "Full precision for model (--no-half)", None, None, None), "no_half_vae": OptionInfo(False if not cmd_opts.use_openvino else True, "Full precision for VAE (--no-half-vae)"), "upcast_sampling": OptionInfo(False if sys.platform != "darwin" else True, "Upcast sampling"), @@ -415,7 +418,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "nan_skip": OptionInfo(False, "Skip Generation if NaN found in latents", gr.Checkbox, {"visible": True}), "rollback_vae": OptionInfo(False, "Attempt VAE roll back for NaN values"), - "cross_attention_sep": OptionInfo("

Attention

", "", gr.HTML), + "cross_attention_sep": OptionInfo("

Cross Attention

", "", gr.HTML), "cross_attention_optimization": OptionInfo(cross_attention_optimization_default, "Attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention(native) }), "sdp_options": OptionInfo(sdp_options_default, "SDP options", gr.CheckboxGroup, {"choices": ['Flash attention', 'Memory attention', 'Math attention'] }), "xformers_options": OptionInfo(['Flash attention'], "xFormers options", gr.CheckboxGroup, {"choices": ['Flash attention'] }), @@ -425,10 +428,9 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "sub_quad_kv_chunk_size": OptionInfo(512, "Attention kv chunk size", gr.Slider, {"minimum": 0, "maximum": 8192, "step": 8, "visible": not native}), "sub_quad_chunk_threshold": OptionInfo(80, "Attention chunking threshold", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1, "visible": not native}), - "other_sep": OptionInfo("

Execution precision

", "", gr.HTML), + "other_sep": OptionInfo("

Execution options

", "", gr.HTML), "opt_channelslast": OptionInfo(False, "Use channels last "), "cudnn_benchmark": OptionInfo(False, "Full-depth cuDNN benchmark feature"), - "cudnn_deterministic": OptionInfo(False, "Use deterministic options for cuDNN"), "diffusers_fuse_projections": OptionInfo(False, "Fused projections"), "torch_gc_threshold": OptionInfo(80, "Torch memory threshold for GC", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}), "torch_malloc": OptionInfo("native", "Torch memory allocator", gr.Radio, {"choices": ['native', 'cudaMallocAsync'] }), From 1a829efcf53e84ece4bd3ff1e80e861547288823 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 7 Jun 2024 09:27:16 -0400 Subject: [PATCH 23/34] fix hidifussion for sd15 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a698338a..882a3c111 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ - improve metadata/infotext parser add `cli/image-exif.py` that can be used to view/extract metadata from images - auto-synchronize modernui and core branches +- fix apply/unapply hidiffusion for sd15 ## Update for 2024-06-02 From 0680a88be3296921236bbe2e7c86d869dae0db46 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 7 Jun 2024 09:59:48 -0400 Subject: [PATCH 24/34] add resadapter --- CHANGELOG.md | 3 +++ TODO.md | 2 -- scripts/resadapter.py | 57 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 scripts/resadapter.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 882a3c111..13c1b8b2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,9 @@ compatible with *SD15* - **PCM LoRAs** allow for fast denoising using less steps with standard *SD15* and *SDXL* models download from +- [ByteDance ResAdapter](https://github.com/bytedance/res-adapter) resolution-free model adapter + allows to use resolutions from 0.5 to 2.0 of original model resolution, compatible with *SD15* and *SDXL* + enable via scripts -> resadapter and select desired model - **Kohya HiRes Fix** allows for higher resolution generation using standard *SD15* models enable via scripts -> kohya-hires-fix *note*: alternative to regular hidiffusion method, but with different approach to scaling diff --git a/TODO.md b/TODO.md index 8e06431de..e2159feb0 100644 --- a/TODO.md +++ b/TODO.md @@ -5,7 +5,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Future Candidates - stable diffusion 3.0: unreleased -- boxdiff - animatediff-sdxl - async lowvram: - fp8: @@ -14,7 +13,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - diffusers public callbacks - include reference styles - lora: sc lora, dora, etc -- resadapter: ## Experimental diff --git a/scripts/resadapter.py b/scripts/resadapter.py new file mode 100644 index 000000000..a70967320 --- /dev/null +++ b/scripts/resadapter.py @@ -0,0 +1,57 @@ +from safetensors.torch import load_file +from huggingface_hub import hf_hub_download +import gradio as gr +from modules import scripts, processing, shared, sd_models, devices + + +repo = 'jiaxiangc/res-adapter' +models = { + 'None': '', + 'SD15 v2 general': 'resadapter_v2_sd1.5', + 'SDXL v2 general': 'resadapter_v2_sdxl', + 'SD15 v1 general': 'resadapter_v1_sd1.5', + 'SD15 v1 extrapolation': 'resadapter_v1_sd1.5_extrapolation', + 'SD15 v1 interpolation': 'resadapter_v1_sd1.5_interpolation', + 'SDXL v1 general': 'resadapter_v1_sdxl', + 'SDXL v1 extrapolation': 'resadapter_v1_sdxl_extrapolation', + 'SDXL v1 interpolation': 'resadapter_v1_sdxl_interpolation', +} + +class Script(scripts.Script): + def title(self): + return 'ResAdapter' + + def show(self, is_img2img): + return not is_img2img if shared.native else False + + # return signature is array of gradio components + def ui(self, _is_img2img): + with gr.Row(): + gr.HTML('  ResAdapter
') + with gr.Row(): + model = gr.Dropdown(label="Model", choices=list(models), value="None") + weight = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label="Weight", value=1.0) + return [model, weight] + + def run(self, p: processing.StableDiffusionProcessing, model, weight): # pylint: disable=arguments-differ + if not shared.native or model == 'None': + return None + if shared.sd_model_type == 'sd': + if not model.startswith('SD15'): + shared.log.warning(f'ResAdapter: pipeline={shared.sd_model_type} selected={model}') + return None + if shared.sd_model_type == 'sdxl': + if not model.startswith('SDXL'): + shared.log.warning(f'ResAdapter: pipeline={shared.sd_model_type} selected={model}') + return None + + old_pipe = shared.sd_model + shared.sd_model.load_lora_weights(hf_hub_download(repo_id=repo, subfolder=models[model], filename="pytorch_lora_weights.safetensors"), adapter_name="res_adapter") + shared.sd_model.set_adapters(["res_adapter"], adapter_weights=[weight]) + shared.sd_model.unet.load_state_dict(load_file(hf_hub_download(repo_id=repo, subfolder=models[model], filename="diffusion_pytorch_model.safetensors")), strict=False) + sd_models.move_model(shared.sd_model, devices.device) # move pipeline to device + sd_models.set_diffuser_options(shared.sd_model, vae=None, op='model') + shared.log.debug(f'ResAdapter: pipeline={shared.sd_model.__class__.__name__} model="{model}" weight={weight} fn={models[model]}') + processed = processing.process_images(p) + shared.sd_model = old_pipe + return processed From d4fead09a6322170ff1198b1c4b33527f4466e71 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 8 Jun 2024 13:37:52 -0400 Subject: [PATCH 25/34] refactor initial installer --- CHANGELOG.md | 21 +- installer.py | 447 +++++++++++++++------------- launch.py | 31 +- modules/ipadapter.py | 2 +- modules/postprocess/gfpgan_model.py | 4 +- webui.sh | 5 +- 6 files changed, 280 insertions(+), 230 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13c1b8b2b..6ed274e84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,18 @@ - StableDiffusion 3 -## Update for 2024-06-04 +## Update for 2024-06-08 *Note*: New features require `diffusers==0.29.0.dev` +### New Models + - [Tenecent HunyuanDiT](https://github.com/Tencent/HunyuanDiT) bilingual english/chinese diffusion transformer model note: this is a very large model at ~17GB, but can be used with less VRAM using model offloading simply select from networks -> models -> reference, model will be auto-downloaded on first use + +### New Functionality + - [MuLan](https://github.com/mulanai/MuLan) Multi-langunage prompts write your prompts forin ~110 auto-detected languages! compatible with *SD15* and *SDXL* @@ -44,13 +49,21 @@ - add torch **full deterministic mode** enable in settings -> compute -> use deterministic mode typical differences are not large and its disabled by default as it does have some performance impact -- lower overhead on generate calls -- cumulative fixes since the last release -- add python version check for torch-directml + +### Improvements + +- improved **installer** for initial installs + initial install will do single-pass install of all required packages with correct versions + subsequent runs will check package versions as necessary +- add env variable `SD_PIP_DEBUG` to write `pip.log` for all pip operations + also improved installer logging +- add python version check for `torch-directml` - improve metadata/infotext parser add `cli/image-exif.py` that can be used to view/extract metadata from images +- lower overhead on generate calls - auto-synchronize modernui and core branches - fix apply/unapply hidiffusion for sd15 +- cumulative fixes since the last release ## Update for 2024-06-02 diff --git a/installer.py b/installer.py index f965c76d9..5de54feea 100644 --- a/installer.py +++ b/installer.py @@ -26,6 +26,7 @@ version = None current_branch = None log = logging.getLogger("sd") debug = log.debug if os.environ.get('SD_INSTALL_DEBUG', None) is not None else lambda *args, **kwargs: None +pip_log = '--log pip.log ' if os.environ.get('SD_PIP_DEBUG', None) is not None else '' log_file = os.path.join(os.path.dirname(__file__), 'sdnext.log') log_rolled = False first_call = True @@ -84,7 +85,10 @@ def setup_logging(): def get(self): return self.buffer - install('rich', 'rich') + install('rich', 'rich', quiet=True) + install('setuptools==69.5.1', 'setuptools', quiet=True) + install('psutil', 'psutil', quiet=True) + install('requests', 'requests', quiet=True) from functools import partial, partialmethod from logging.handlers import RotatingFileHandler from rich.theme import Theme @@ -233,11 +237,11 @@ def uninstall(package, quiet = False): @lru_cache() def pip(arg: str, ignore: bool = False, quiet: bool = False): arg = arg.replace('>=', '==') - if not quiet: - log.info(f'Installing package: {arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace(" ", " ").strip()}') + if not quiet and '-r ' not in arg: + log.info(f'Install: package="{arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace(" ", " ").strip()}"') env_args = os.environ.get("PIP_EXTRA_ARGS", "") - log.debug(f"Running pip: {arg} {env_args}") - result = subprocess.run(f'"{sys.executable}" -m pip {arg} {env_args}', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + log.debug(f'Running: pip="{pip_log}{arg} {env_args}"') + result = subprocess.run(f'"{sys.executable}" -m pip {pip_log}{arg} {env_args}', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE) txt = result.stdout.decode(encoding="utf8", errors="ignore") if len(result.stderr) > 0: txt += ('\n' if len(txt) > 0 else '') + result.stderr.decode(encoding="utf8", errors="ignore") @@ -253,14 +257,14 @@ def pip(arg: str, ignore: bool = False, quiet: bool = False): # install package using pip if not already installed @lru_cache() -def install(package, friendly: str = None, ignore: bool = False, reinstall: bool = False, no_deps: bool = False): +def install(package, friendly: str = None, ignore: bool = False, reinstall: bool = False, no_deps: bool = False, quiet: bool = False): res = '' if args.reinstall or args.upgrade: global quick_allowed # pylint: disable=global-statement quick_allowed = False - if args.reinstall or reinstall or not installed(package, friendly, quiet=False): - deps = '' if not no_deps else '--no-deps' - res = pip(f"install --upgrade {deps} {package}", ignore=ignore) + if args.reinstall or reinstall or not installed(package, friendly, quiet=quiet): + deps = '' if not no_deps else '--no-deps ' + res = pip(f"install --upgrade {deps}{package}", ignore=ignore) try: import imp # pylint: disable=deprecated-module imp.reload(pkg_resources) @@ -388,10 +392,10 @@ def get_platform(): # check python version -def check_python(supported_minors=[9, 10, 11], reason=None): +def check_python(supported_minors=[9, 10, 11, 12], reason=None): if args.quick: return - log.info(f'Python {platform.python_version()} on {platform.system()}') + log.info(f'Python version={platform.python_version()} platform={platform.system()} bin="{sys.executable}" venv="{sys.prefix}"') if not (int(sys.version_info.major) == 3 and int(sys.version_info.minor) in supported_minors): log.error(f"Incompatible Python version: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro} required 3.{supported_minors}") if reason is not None: @@ -427,6 +431,212 @@ def check_onnx(): install('onnxruntime', 'onnxruntime', ignore=True) +def install_rocm_zluda(torch_command): + is_windows = platform.system() == 'Windows' + log.info('AMD ROCm toolkit detected') + os.environ.setdefault('PYTORCH_HIP_ALLOC_CONF', 'garbage_collection_threshold:0.8,max_split_size_mb:512') + if not is_windows: + os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow-rocm') + try: + if is_windows: + command = subprocess.run('hipinfo', shell=True, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + amd_gpus = command.stdout.decode(encoding="utf8", errors="ignore").split('\n') + amd_gpus = [x.split(' ')[-1].strip() for x in amd_gpus if x.startswith('gcnArchName:')] + else: + command = subprocess.run('rocm_agent_enumerator', shell=True, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + amd_gpus = command.stdout.decode(encoding="utf8", errors="ignore").split('\n') + amd_gpus = [x for x in amd_gpus if x and x != 'gfx000'] + log.debug(f'ROCm agents detected: {amd_gpus}') + except Exception as e: + log.debug(f'ROCm agent enumerator failed: {e}') + amd_gpus = [] + + hip_visible_devices = [] # use the first available amd gpu by default + for idx, gpu in enumerate(amd_gpus): + if gpu in ['gfx1100', 'gfx1101', 'gfx1102']: + hip_visible_devices.append((idx, gpu, 'navi3x')) + break + if gpu in ['gfx1030', 'gfx1031', 'gfx1032', 'gfx1034']: # experimental navi 2x support + hip_visible_devices.append((idx, gpu, 'navi2x')) + break + if len(hip_visible_devices) > 0: + idx, gpu, arch = hip_visible_devices[0] + log.debug(f'ROCm agent used by default: idx={idx} gpu={gpu} arch={arch}') + os.environ.setdefault('HIP_VISIBLE_DEVICES', str(idx)) + if arch == 'navi3x': + os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', '11.0.0') + if os.environ.get('TENSORFLOW_PACKAGE') == 'tensorflow-rocm': # do not use tensorflow-rocm for navi 3x + os.environ['TENSORFLOW_PACKAGE'] = 'tensorflow==2.13.0' + elif arch == 'navi2x': + os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', '10.3.0') + else: + log.debug(f'HSA_OVERRIDE_GFX_VERSION auto config is skipped for {gpu}') + try: + command = subprocess.run('hipconfig --version', shell=True, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + arr = command.stdout.decode(encoding="utf8", errors="ignore").split('.') + rocm_ver = f'{arr[0]}.{arr[1]}' if len(arr) >= 2 else None + log.debug(f'ROCm version detected: {rocm_ver}') + except Exception as e: + log.debug(f'ROCm hipconfig failed: {e}') + rocm_ver = None + if args.use_zluda: + log.warning("ZLUDA support: experimental") + error = None + from modules import zluda_installer + try: + if args.reinstall_zluda: + zluda_installer.uninstall() + if args.experimental: + zluda_installer.enable_runtime_api() + zluda_path = zluda_installer.get_path() + zluda_installer.install(zluda_path) + zluda_installer.make_copy(zluda_path) + except Exception as e: + error = e + log.warning(f'Failed to install ZLUDA: {e}') + if error is None: + try: + zluda_installer.load(zluda_path) + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.3.0 torchvision --index-url https://download.pytorch.org/whl/cu118') + log.info(f'Using ZLUDA in {zluda_path}') + except Exception as e: + error = e + log.warning(f'Failed to load ZLUDA: {e}') + if error is not None: + log.info('Using CPU-only torch') + torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') + elif is_windows: # TODO TBD after ROCm for Windows is released + log.warning("HIP SDK is detected, but no Torch release for Windows available") + log.info("For ZLUDA support specify '--use-zluda'") + log.info('Using CPU-only torch') + torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') + + # conceal ROCm installed + os.environ.pop("ROCM_HOME", None) + os.environ.pop("ROCM_PATH", None) + paths = os.environ["PATH"].split(";") + paths_no_rocm = [] + for path in paths: + if "ROCm" not in path: + paths_no_rocm.append(path) + os.environ["PATH"] = ";".join(paths_no_rocm) + else: + if rocm_ver is None: # assume the latest if version check fails + torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/rocm6.0') + elif rocm_ver == "6.1": # need nightlies + torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --pre --index-url https://download.pytorch.org/whl/nightly/rocm6.1') + elif float(rocm_ver) < 5.5: # oldest supported version is 5.5 + log.warning(f"Unsupported ROCm version detected: {rocm_ver}") + log.warning("Minimum supported ROCm version is 5.5") + torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/rocm5.5') + else: + torch_command = os.environ.get('TORCH_COMMAND', f'torch torchvision --index-url https://download.pytorch.org/whl/rocm{rocm_ver}') + if rocm_ver is not None: + ort_version = os.environ.get('ONNXRUNTIME_VERSION', None) + ort_package = os.environ.get('ONNXRUNTIME_PACKAGE', f"--pre onnxruntime-training{'' if ort_version is None else ('==' + ort_version)} --index-url https://pypi.lsh.sh/{rocm_ver[0]}{rocm_ver[2]} --extra-index-url https://pypi.org/simple") + install(ort_package, 'onnxruntime-training') + return torch_command + + +def install_ipex(torch_command): + args.use_ipex = True # pylint: disable=attribute-defined-outside-init + log.info('Intel OneAPI Toolkit detected') + if os.environ.get("NEOReadDebugKeys", None) is None: + os.environ.setdefault('NEOReadDebugKeys', '1') + if os.environ.get("ClDeviceGlobalMemSizeAvailablePercent", None) is None: + os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100') + if "linux" in sys.platform: + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.1.0.post0 torchvision==0.16.0.post0 intel-extension-for-pytorch==2.1.20+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/') + os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow==2.15.0 intel-extension-for-tensorflow[xpu]==2.15.0.0') + if os.environ.get('DISABLE_VENV_LIBS', None) is None: + install(os.environ.get('MKL_PACKAGE', 'mkl==2024.1.0'), 'mkl') + install(os.environ.get('DPCPP_PACKAGE', 'mkl-dpcpp==2024.1.0'), 'mkl-dpcpp') + install(os.environ.get('ONECCL_PACKAGE', 'oneccl-devel==2021.12.0'), 'oneccl-devel') + install(os.environ.get('MPI_PACKAGE', 'impi-devel==2021.12.0'), 'impi-devel') + else: + if sys.version_info.minor == 11: + pytorch_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/torch-2.1.0a0+cxx11.abi-cp311-cp311-win_amd64.whl' + torchvision_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/torchvision-0.16.0a0+cxx11.abi-cp311-cp311-win_amd64.whl' + ipex_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/intel_extension_for_pytorch-2.1.10+xpu-cp311-cp311-win_amd64.whl' + torch_command = os.environ.get('TORCH_COMMAND', f'{pytorch_pip} {torchvision_pip} {ipex_pip}') + elif sys.version_info.minor == 10: + pytorch_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/torch-2.1.0a0+cxx11.abi-cp310-cp310-win_amd64.whl' + torchvision_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/torchvision-0.16.0a0+cxx11.abi-cp310-cp310-win_amd64.whl' + ipex_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/intel_extension_for_pytorch-2.1.10+xpu-cp310-cp310-win_amd64.whl' + torch_command = os.environ.get('TORCH_COMMAND', f'{pytorch_pip} {torchvision_pip} {ipex_pip}') + else: + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.1.0.post0 torchvision==0.16.0.post0 intel-extension-for-pytorch==2.1.20+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/') + if os.environ.get('DISABLE_VENV_LIBS', None) is None: + install(os.environ.get('MKL_PACKAGE', 'mkl==2024.1.0'), 'mkl') + install(os.environ.get('DPCPP_PACKAGE', 'mkl-dpcpp==2024.1.0'), 'mkl-dpcpp') + install(os.environ.get('ONECCL_PACKAGE', 'oneccl-devel==2021.12.0'), 'oneccl-devel') + install(os.environ.get('MPI_PACKAGE', 'impi-devel==2021.12.0'), 'impi-devel') + torch_command = os.environ.get('TORCH_COMMAND', f'{pytorch_pip} {torchvision_pip} {ipex_pip}') + install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2023.3.0'), 'openvino', ignore=True) + install('nncf==2.7.0', 'nncf', ignore=True) + install(os.environ.get('ONNXRUNTIME_PACKAGE', 'onnxruntime-openvino'), 'onnxruntime-openvino', ignore=True) + return torch_command + + +def install_openvino(torch_command): + log.info('Using OpenVINO') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.2.0 torchvision==0.17.0 --index-url https://download.pytorch.org/whl/cpu') + install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2023.3.0'), 'openvino') + install(os.environ.get('ONNXRUNTIME_PACKAGE', 'onnxruntime-openvino'), 'onnxruntime-openvino', ignore=True) + install('nncf==2.8.1', 'nncf') + os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX') + if os.environ.get("NEOReadDebugKeys", None) is None: + os.environ.setdefault('NEOReadDebugKeys', '1') + if os.environ.get("ClDeviceGlobalMemSizeAvailablePercent", None) is None: + os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100') + return torch_command + + +def is_rocm_available(allow_rocm): + if not allow_rocm: + return False + if installed('torch-directml', quiet=True): + log.debug('DirectML installation is detected. Skipping HIP SDK check.') + return False + if platform.system() == 'Windows': + from modules.zluda_installer import find_hip_sdk + return find_hip_sdk() is not None + else: + return shutil.which('rocminfo') is not None or os.path.exists('/opt/rocm/bin/rocminfo') or os.path.exists('/dev/kfd') + + +def install_torch_addons(): + xformers_package = os.environ.get('XFORMERS_PACKAGE', '--pre xformers') if opts.get('cross_attention_optimization', '') == 'xFormers' or args.use_xformers else 'none' + triton_command = os.environ.get('TRITON_COMMAND', 'triton') if sys.platform == 'linux' else None + if 'xformers' in xformers_package: + try: + install(f'--no-deps {xformers_package}', ignore=True) + import torch # pylint: disable=unused-import + import xformers # pylint: disable=unused-import + except Exception as e: + log.debug(f'Cannot install xformers package: {e}') + elif not args.experimental and not args.use_xformers and opts.get('cross_attention_optimization', '') != 'xFormers': + uninstall('xformers') + if opts.get('cuda_compile_backend', '') == 'hidet': + install('hidet', 'hidet') + if opts.get('cuda_compile_backend', '') == 'deep-cache': + install('DeepCache') + if opts.get('cuda_compile_backend', '') == 'olive-ai': + install('olive-ai') + if opts.get('nncf_compress_weights', False) and not args.use_openvino: + install('nncf==2.7.0', 'nncf') + if triton_command is not None: + install(triton_command, 'triton', quiet=True) + + +def is_cuda_available(allow_cuda): + return allow_cuda and (shutil.which('nvidia-smi') is not None or args.use_xformers or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe'))) + + +def is_ipex_available(allow_ipex): + return allow_ipex and (args.use_ipex or shutil.which('sycl-ls') is not None or shutil.which('sycl-ls.exe') is not None or os.environ.get('ONEAPI_ROOT') is not None or os.path.exists('/opt/intel/oneapi') or os.path.exists("C:/Program Files (x86)/Intel/oneAPI") or os.path.exists("C:/oneAPI")) + + # check torch version def check_torch(): if args.skip_torch: @@ -443,179 +653,19 @@ def check_torch(): log.debug(f'Torch overrides: cuda={args.use_cuda} rocm={args.use_rocm} ipex={args.use_ipex} diml={args.use_directml} openvino={args.use_openvino}') log.debug(f'Torch allowed: cuda={allow_cuda} rocm={allow_rocm} ipex={allow_ipex} diml={allow_directml} openvino={allow_openvino}') torch_command = os.environ.get('TORCH_COMMAND', '') - xformers_package = os.environ.get('XFORMERS_PACKAGE', '--pre xformers') if opts.get('cross_attention_optimization', '') == 'xFormers' or args.use_xformers else 'none' - triton_command = os.environ.get('TRITON_COMMAND', 'triton') if sys.platform == 'linux' else None - - def is_rocm_available(): - if not allow_rocm: - return False - if installed('torch-directml', quiet=True): - log.debug('DirectML installation is detected. Skipping HIP SDK check.') - return False - if platform.system() == 'Windows': - from modules.zluda_installer import find_hip_sdk - return find_hip_sdk() is not None - else: - return shutil.which('rocminfo') is not None or os.path.exists('/opt/rocm/bin/rocminfo') or os.path.exists('/dev/kfd') if torch_command != '': pass - elif allow_cuda and (shutil.which('nvidia-smi') is not None or args.use_xformers or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe'))): + elif is_cuda_available(allow_cuda): log.info('nVidia CUDA toolkit detected: nvidia-smi present') torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/cu121') - install('onnxruntime-gpu', 'onnxruntime-gpu', ignore=True) - elif is_rocm_available(): - is_windows = platform.system() == 'Windows' - log.info('AMD ROCm toolkit detected') - os.environ.setdefault('PYTORCH_HIP_ALLOC_CONF', 'garbage_collection_threshold:0.8,max_split_size_mb:512') - if not is_windows: - os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow-rocm') - try: - if is_windows: - command = subprocess.run('hipinfo', shell=True, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - amd_gpus = command.stdout.decode(encoding="utf8", errors="ignore").split('\n') - amd_gpus = [x.split(' ')[-1].strip() for x in amd_gpus if x.startswith('gcnArchName:')] - else: - command = subprocess.run('rocm_agent_enumerator', shell=True, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - amd_gpus = command.stdout.decode(encoding="utf8", errors="ignore").split('\n') - amd_gpus = [x for x in amd_gpus if x and x != 'gfx000'] - log.debug(f'ROCm agents detected: {amd_gpus}') - except Exception as e: - log.debug(f'ROCm agent enumerator failed: {e}') - amd_gpus = [] - - hip_visible_devices = [] # use the first available amd gpu by default - for idx, gpu in enumerate(amd_gpus): - if gpu in ['gfx1100', 'gfx1101', 'gfx1102']: - hip_visible_devices.append((idx, gpu, 'navi3x')) - break - if gpu in ['gfx1030', 'gfx1031', 'gfx1032', 'gfx1034']: # experimental navi 2x support - hip_visible_devices.append((idx, gpu, 'navi2x')) - break - if len(hip_visible_devices) > 0: - idx, gpu, arch = hip_visible_devices[0] - log.debug(f'ROCm agent used by default: idx={idx} gpu={gpu} arch={arch}') - os.environ.setdefault('HIP_VISIBLE_DEVICES', str(idx)) - if arch == 'navi3x': - os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', '11.0.0') - if os.environ.get('TENSORFLOW_PACKAGE') == 'tensorflow-rocm': # do not use tensorflow-rocm for navi 3x - os.environ['TENSORFLOW_PACKAGE'] = 'tensorflow==2.13.0' - elif arch == 'navi2x': - os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', '10.3.0') - else: - log.debug(f'HSA_OVERRIDE_GFX_VERSION auto config is skipped for {gpu}') - try: - command = subprocess.run('hipconfig --version', shell=True, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - arr = command.stdout.decode(encoding="utf8", errors="ignore").split('.') - rocm_ver = f'{arr[0]}.{arr[1]}' if len(arr) >= 2 else None - log.debug(f'ROCm version detected: {rocm_ver}') - except Exception as e: - log.debug(f'ROCm hipconfig failed: {e}') - rocm_ver = None - if args.use_zluda: - log.warning("ZLUDA support: experimental") - error = None - from modules import zluda_installer - try: - if args.reinstall_zluda: - zluda_installer.uninstall() - if args.experimental: - zluda_installer.enable_runtime_api() - zluda_path = zluda_installer.get_path() - zluda_installer.install(zluda_path) - zluda_installer.make_copy(zluda_path) - except Exception as e: - error = e - log.warning(f'Failed to install ZLUDA: {e}') - if error is None: - try: - zluda_installer.load(zluda_path) - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.3.0 torchvision --index-url https://download.pytorch.org/whl/cu118') - log.info(f'Using ZLUDA in {zluda_path}') - except Exception as e: - error = e - log.warning(f'Failed to load ZLUDA: {e}') - if error is not None: - log.info('Using CPU-only torch') - torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') - elif is_windows: # TODO TBD after ROCm for Windows is released - log.warning("HIP SDK is detected, but no Torch release for Windows available") - log.info("For ZLUDA support specify '--use-zluda'") - log.info('Using CPU-only torch') - torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') - - # conceal ROCm installed - os.environ.pop("ROCM_HOME", None) - os.environ.pop("ROCM_PATH", None) - paths = os.environ["PATH"].split(";") - paths_no_rocm = [] - for path in paths: - if "ROCm" not in path: - paths_no_rocm.append(path) - os.environ["PATH"] = ";".join(paths_no_rocm) - else: - if rocm_ver is None: # assume the latest if version check fails - torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/rocm6.0') - elif rocm_ver == "6.1": # need nightlies - torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --pre --index-url https://download.pytorch.org/whl/nightly/rocm6.1') - elif float(rocm_ver) < 5.5: # oldest supported version is 5.5 - log.warning(f"Unsupported ROCm version detected: {rocm_ver}") - log.warning("Minimum supported ROCm version is 5.5") - torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/rocm5.5') - else: - torch_command = os.environ.get('TORCH_COMMAND', f'torch torchvision --index-url https://download.pytorch.org/whl/rocm{rocm_ver}') - if rocm_ver is not None: - ort_version = os.environ.get('ONNXRUNTIME_VERSION', None) - ort_package = os.environ.get('ONNXRUNTIME_PACKAGE', f"--pre onnxruntime-training{'' if ort_version is None else ('==' + ort_version)} --index-url https://pypi.lsh.sh/{rocm_ver[0]}{rocm_ver[2]} --extra-index-url https://pypi.org/simple") - install(ort_package, 'onnxruntime-training') - elif allow_ipex and (args.use_ipex or shutil.which('sycl-ls') is not None or shutil.which('sycl-ls.exe') is not None or os.environ.get('ONEAPI_ROOT') is not None or os.path.exists('/opt/intel/oneapi') or os.path.exists("C:/Program Files (x86)/Intel/oneAPI") or os.path.exists("C:/oneAPI")): - args.use_ipex = True # pylint: disable=attribute-defined-outside-init - log.info('Intel OneAPI Toolkit detected') - if os.environ.get("NEOReadDebugKeys", None) is None: - os.environ.setdefault('NEOReadDebugKeys', '1') - if os.environ.get("ClDeviceGlobalMemSizeAvailablePercent", None) is None: - os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100') - if "linux" in sys.platform: - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.1.0.post0 torchvision==0.16.0.post0 intel-extension-for-pytorch==2.1.20+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/') - os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow==2.15.0 intel-extension-for-tensorflow[xpu]==2.15.0.0') - if os.environ.get('DISABLE_VENV_LIBS', None) is None: - install(os.environ.get('MKL_PACKAGE', 'mkl==2024.1.0'), 'mkl') - install(os.environ.get('DPCPP_PACKAGE', 'mkl-dpcpp==2024.1.0'), 'mkl-dpcpp') - install(os.environ.get('ONECCL_PACKAGE', 'oneccl-devel==2021.12.0'), 'oneccl-devel') - install(os.environ.get('MPI_PACKAGE', 'impi-devel==2021.12.0'), 'impi-devel') - else: - if sys.version_info.minor == 11: - pytorch_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/torch-2.1.0a0+cxx11.abi-cp311-cp311-win_amd64.whl' - torchvision_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/torchvision-0.16.0a0+cxx11.abi-cp311-cp311-win_amd64.whl' - ipex_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/intel_extension_for_pytorch-2.1.10+xpu-cp311-cp311-win_amd64.whl' - torch_command = os.environ.get('TORCH_COMMAND', f'{pytorch_pip} {torchvision_pip} {ipex_pip}') - elif sys.version_info.minor == 10: - pytorch_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/torch-2.1.0a0+cxx11.abi-cp310-cp310-win_amd64.whl' - torchvision_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/torchvision-0.16.0a0+cxx11.abi-cp310-cp310-win_amd64.whl' - ipex_pip = 'https://github.com/Nuullll/intel-extension-for-pytorch/releases/download/v2.1.10%2Bxpu/intel_extension_for_pytorch-2.1.10+xpu-cp310-cp310-win_amd64.whl' - torch_command = os.environ.get('TORCH_COMMAND', f'{pytorch_pip} {torchvision_pip} {ipex_pip}') - else: - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.1.0.post0 torchvision==0.16.0.post0 intel-extension-for-pytorch==2.1.20+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/') - if os.environ.get('DISABLE_VENV_LIBS', None) is None: - install(os.environ.get('MKL_PACKAGE', 'mkl==2024.1.0'), 'mkl') - install(os.environ.get('DPCPP_PACKAGE', 'mkl-dpcpp==2024.1.0'), 'mkl-dpcpp') - install(os.environ.get('ONECCL_PACKAGE', 'oneccl-devel==2021.12.0'), 'oneccl-devel') - install(os.environ.get('MPI_PACKAGE', 'impi-devel==2021.12.0'), 'impi-devel') - torch_command = os.environ.get('TORCH_COMMAND', f'{pytorch_pip} {torchvision_pip} {ipex_pip}') - install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2023.3.0'), 'openvino', ignore=True) - install('nncf==2.7.0', 'nncf', ignore=True) - install(os.environ.get('ONNXRUNTIME_PACKAGE', 'onnxruntime-openvino'), 'onnxruntime-openvino', ignore=True) + install('onnxruntime-gpu', 'onnxruntime-gpu', ignore=True, quiet=True) + elif is_rocm_available(allow_rocm): + torch_command = install_rocm_zluda(torch_command) + elif is_ipex_available(allow_ipex): + torch_command = install_ipex(torch_command) elif allow_openvino and args.use_openvino: - log.info('Using OpenVINO') - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.2.0 torchvision==0.17.0 --index-url https://download.pytorch.org/whl/cpu') - install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2023.3.0'), 'openvino') - install(os.environ.get('ONNXRUNTIME_PACKAGE', 'onnxruntime-openvino'), 'onnxruntime-openvino', ignore=True) - install('nncf==2.8.1', 'nncf') - os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX') - if os.environ.get("NEOReadDebugKeys", None) is None: - os.environ.setdefault('NEOReadDebugKeys', '1') - if os.environ.get("ClDeviceGlobalMemSizeAvailablePercent", None) is None: - os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100') + torch_command = install_openvino(torch_command) else: machine = platform.machine() if sys.platform == 'darwin': @@ -633,11 +683,7 @@ def check_torch(): log.info('Using CPU-only Torch') torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') if 'torch' in torch_command and not args.version: - if not installed('torch', quiet=True): - log.debug(f'Installing torch: {torch_command}') - install(torch_command, 'torch torchvision') - if triton_command is not None: - install(triton_command, 'triton') + install(torch_command, 'torch torchvision', quiet=True) else: try: import torch @@ -676,23 +722,7 @@ def check_torch(): if args.version: return if not args.skip_all: - try: - if 'xformers' in xformers_package: - install(f'--no-deps {xformers_package}', ignore=True) - import torch - import xformers # pylint: disable=unused-import - elif not args.experimental and not args.use_xformers and opts.get('cross_attention_optimization', '') != 'xFormers': - uninstall('xformers') - except Exception as e: - log.debug(f'Cannot install xformers package: {e}') - if opts.get('cuda_compile_backend', '') == 'hidet': - install('hidet', 'hidet') - if opts.get('cuda_compile_backend', '') == 'deep-cache': - install('DeepCache') - if opts.get('cuda_compile_backend', '') == 'olive-ai': - install('olive-ai') - if opts.get('nncf_compress_weights', False) and not args.use_openvino: - install('nncf==2.7.0', 'nncf') + install_torch_addons() if args.profile: print_profile(pr, 'Torch') @@ -724,12 +754,12 @@ def install_packages(): pr.enable() log.info('Verifying packages') clip_package = os.environ.get('CLIP_PACKAGE', "git+https://github.com/openai/CLIP.git") - install(clip_package, 'clip') + install(clip_package, 'clip', quiet=True) tensorflow_package = os.environ.get('TENSORFLOW_PACKAGE', 'tensorflow==2.13.0') - install(tensorflow_package, 'tensorflow-rocm' if 'rocm' in tensorflow_package else 'tensorflow', ignore=True) + install(tensorflow_package, 'tensorflow-rocm' if 'rocm' in tensorflow_package else 'tensorflow', ignore=True, quiet=True) bitsandbytes_package = os.environ.get('BITSANDBYTES_PACKAGE', None) if bitsandbytes_package is not None: - install(bitsandbytes_package, 'bitsandbytes', ignore=True) + install(bitsandbytes_package, 'bitsandbytes', ignore=True, quiet=True) elif not args.experimental: uninstall('bitsandbytes') if args.profile: @@ -874,11 +904,18 @@ def install_requirements(): pr.enable() if args.skip_requirements and not args.requirements: return + if not installed('diffusers', quiet=True): # diffusers are not installed, so run initial installation + global quick_allowed # pylint: disable=global-statement + quick_allowed = False + log.info('Installing requirements: this make take a while...') + pip('install -r requirements.txt') + installed('torch', reload=True) # reload packages cache log.info('Verifying requirements') with open('requirements.txt', 'r', encoding='utf8') as f: lines = [line.strip() for line in f.readlines() if line.strip() != '' and not line.startswith('#') and line is not None] for line in lines: - _res = install(line) + if not installed(line, quiet=True): + _res = install(line) if args.profile: print_profile(pr, 'Requirements') @@ -909,7 +946,7 @@ def set_environment(): os.environ.setdefault('KINETO_LOG_LEVEL', '3') os.environ.setdefault('DO_NOT_TRACK', '1') os.environ.setdefault('HF_HUB_CACHE', opts.get('hfcache_dir', os.path.join(os.path.expanduser('~'), '.cache', 'huggingface', 'hub'))) - log.debug(f'HF cache folder: {os.environ.get("HF_HUB_CACHE")}') + log.info(f'HF cache folder: {os.environ.get("HF_HUB_CACHE")}') allocator = f'garbage_collection_threshold:{opts.get("torch_gc_threshold", 80)/100:0.2f},max_split_size_mb:512' if opts.get("torch_malloc", "native") == 'cudaMallocAsync': allocator += ',backend:cudaMallocAsync' diff --git a/launch.py b/launch.py index d8da5f5bb..f1d8b7ec5 100755 --- a/launch.py +++ b/launch.py @@ -215,26 +215,25 @@ def main(): installer.log.info('Startup: skip all') installer.quick_allowed = True init_paths() - elif installer.check_timestamp(): - installer.log.info('Startup: quick launch') - installer.install_requirements() - installer.install_packages() - init_paths() - installer.check_extensions() else: - installer.log.info('Startup: standard') installer.install_requirements() installer.install_packages() - installer.install_submodules() - init_paths() - installer.install_extensions() - installer.install_requirements() # redo requirements since extensions may change them - installer.update_wiki() - if installer.errors == 0: - installer.log.debug(f'Setup complete without errors: {round(time.time())}') + if installer.check_timestamp(): + installer.log.info('Startup: quick launch') + init_paths() + installer.check_extensions() else: - installer.log.warning(f'Setup complete with errors: {installer.errors}') - installer.log.warning(f'See log file for more details: {installer.log_file}') + installer.log.info('Startup: standard') + installer.install_submodules() + init_paths() + installer.install_extensions() + installer.install_requirements() # redo requirements since extensions may change them + installer.update_wiki() + if installer.errors == 0: + installer.log.debug(f'Setup complete without errors: {round(time.time())}') + else: + installer.log.warning(f'Setup complete with errors: {installer.errors}') + installer.log.warning(f'See log file for more details: {installer.log_file}') installer.extensions_preload(parser) # adds additional args from extensions args = installer.parse_args(parser) diff --git a/modules/ipadapter.py b/modules/ipadapter.py index dd073e722..f1fa60197 100644 --- a/modules/ipadapter.py +++ b/modules/ipadapter.py @@ -76,7 +76,7 @@ def unapply(pipe): # pylint: disable=arguments-differ try: if hasattr(pipe, 'set_ip_adapter_scale'): pipe.set_ip_adapter_scale(0) - if hasattr(pipe, 'unet') and hasattr(pipe.unet, 'config')and pipe.unet.config.encoder_hid_dim_type == 'ip_image_proj': + if hasattr(pipe, 'unet') and hasattr(pipe.unet, 'config') and pipe.unet.config.encoder_hid_dim_type == 'ip_image_proj': pipe.unet.encoder_hid_proj = None pipe.config.encoder_hid_dim_type = None pipe.unet.set_default_attn_processor() diff --git a/modules/postprocess/gfpgan_model.py b/modules/postprocess/gfpgan_model.py index fb7ff0f5d..0b06325ed 100644 --- a/modules/postprocess/gfpgan_model.py +++ b/modules/postprocess/gfpgan_model.py @@ -72,8 +72,8 @@ def setup_model(dirname): except Exception: pass try: - install('basicsr') - install('gfpgan') + install('basicsr', quiet=True) + install('gfpgan', quiet=True) import gfpgan import facexlib import modules.face_restoration diff --git a/webui.sh b/webui.sh index 2c863bee5..378aad89e 100755 --- a/webui.sh +++ b/webui.sh @@ -73,8 +73,8 @@ fi if [[ -f "${venv_dir}"/bin/activate ]] then - echo "Activate python venv" source "${venv_dir}"/bin/activate + echo "Activate python venv: $VIRTUAL_ENV" else echo "Error: Cannot activate python venv" exit 1 @@ -103,6 +103,7 @@ then echo "Launch: ipexrun" exec ipexrun --multi-task-manager 'taskset' --memory-allocator 'jemalloc' launch.py "$@" else - echo "Launch" + PYTHON=`which python` + echo "Launch: ${PYTHON}" exec "${PYTHON}" launch.py "$@" fi From 6d6f1de295b836acbe07b3c0d0bdf5f199dcc4fa Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 8 Jun 2024 14:14:48 -0400 Subject: [PATCH 26/34] additional python 3.12 compatibility --- installer.py | 5 +++++ modules/devices.py | 2 +- modules/postprocess/sdupscaler_model.py | 1 - modules/postprocessing.py | 2 +- modules/sd_models.py | 2 -- scripts/face-details.py | 2 +- webui.py | 1 + 7 files changed, 9 insertions(+), 6 deletions(-) diff --git a/installer.py b/installer.py index 5de54feea..9c77a1cf2 100644 --- a/installer.py +++ b/installer.py @@ -402,6 +402,8 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): log.error(reason) if not args.ignore: sys.exit(1) + if int(sys.version_info.minor) == 12: + os.environ.setdefault('SETUPTOOLS_USE_DISTUTILS', 'local') # hack for python 3.11 setuptools if not args.skip_git: git_cmd = os.environ.get('GIT', "git") if shutil.which(git_cmd) is None: @@ -432,6 +434,7 @@ def check_onnx(): def install_rocm_zluda(torch_command): + check_python(supported_minors=[10,11], reason='RocM or Zluda backends require Python 3.10 or 3.11') is_windows = platform.system() == 'Windows' log.info('AMD ROCm toolkit detected') os.environ.setdefault('PYTORCH_HIP_ALLOC_CONF', 'garbage_collection_threshold:0.8,max_split_size_mb:512') @@ -539,6 +542,7 @@ def install_rocm_zluda(torch_command): def install_ipex(torch_command): + check_python(supported_minors=[10,11], reason='IPEX backend requires Python 3.10 or 3.11') args.use_ipex = True # pylint: disable=attribute-defined-outside-init log.info('Intel OneAPI Toolkit detected') if os.environ.get("NEOReadDebugKeys", None) is None: @@ -579,6 +583,7 @@ def install_ipex(torch_command): def install_openvino(torch_command): + check_python(supported_minors=[10,11], reason='IPEX backend requires Python 3.10 or 3.11') log.info('Using OpenVINO') torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.2.0 torchvision==0.17.0 --index-url https://download.pytorch.org/whl/cpu') install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2023.3.0'), 'openvino') diff --git a/modules/devices.py b/modules/devices.py index 8ff3ad04b..675fcac19 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -235,7 +235,7 @@ def set_cuda_params(): torch.use_deterministic_algorithms(shared.opts.cudnn_deterministic) log.debug(f'Torch mode: deterministic={shared.opts.cudnn_deterministic}') if shared.opts.cudnn_deterministic: - os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8' + os.environ.setdefault('CUBLAS_WORKSPACE_CONFIG', ':4096:8') torch.backends.cudnn.benchmark = True if shared.opts.cudnn_benchmark: log.debug('Torch cuDNN: enable benchmark') diff --git a/modules/postprocess/sdupscaler_model.py b/modules/postprocess/sdupscaler_model.py index 0a3106289..e5b1c8b45 100644 --- a/modules/postprocess/sdupscaler_model.py +++ b/modules/postprocess/sdupscaler_model.py @@ -28,7 +28,6 @@ class UpscalerSD(Upscaler): shared.log.debug(f"Upscaler cached: type={scaler.name} model={path}") return self.models[path] else: - devices.set_cuda_params() model = diffusers.DiffusionPipeline.from_pretrained(path, cache_dir=shared.opts.diffusers_dir, torch_dtype=devices.dtype) if hasattr(model, "set_progress_bar_config"): model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + 'Upscale', ncols=80, colour='#327fba') diff --git a/modules/postprocessing.py b/modules/postprocessing.py index 2288401df..346758851 100644 --- a/modules/postprocessing.py +++ b/modules/postprocessing.py @@ -78,7 +78,7 @@ def run_postprocessing(extras_mode, image, image_folder: List[tempfile.NamedTemp pp.image.info[k] = v if 'parameters' in items: info = items['parameters'] + ', ' - info = info + ", ".join([k if k == v else f'{k}: {info.quote(v)}' for k, v in pp.info.items() if v is not None]) + info = info + ", ".join([k if k == v else f'{k}: {infotext.quote(v)}' for k, v in pp.info.items() if v is not None]) pp.image.info["postprocessing"] = info processed_images.append(pp.image) if save_output: diff --git a/modules/sd_models.py b/modules/sd_models.py index f9f89f189..00b0343ba 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -890,7 +890,6 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No timer = Timer() logging.getLogger("diffusers").setLevel(logging.ERROR) timer.record("diffusers") - devices.set_cuda_params() diffusers_load_config = { "low_cpu_mem_usage": True, "torch_dtype": devices.dtype, @@ -1420,7 +1419,6 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None, from modules import sd_hijack_inpainting sd_hijack_inpainting.do_inpainting_hijack() - devices.set_cuda_params() if already_loaded_state_dict is not None: state_dict = already_loaded_state_dict else: diff --git a/scripts/face-details.py b/scripts/face-details.py index f5e554f15..2a53b4234 100644 --- a/scripts/face-details.py +++ b/scripts/face-details.py @@ -32,7 +32,7 @@ class FaceRestorerYolo(FaceRestoration): def dependencies(self): import installer - installer.install('ultralytics', ignore=True) + installer.install('ultralytics', ignore=True, quiet=True) def predict( self, diff --git a/webui.py b/webui.py index fd22fef73..408724eff 100644 --- a/webui.py +++ b/webui.py @@ -156,6 +156,7 @@ def initialize(): def load_model(): + modules.devices.set_cuda_params() if not opts.sd_checkpoint_autoload or (shared.cmd_opts.ckpt is not None and shared.cmd_opts.ckpt.lower() != 'none'): log.debug('Model auto load disabled') else: From ce73566c1d7d6a8f8161851b26573b8d21fb6401 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 8 Jun 2024 15:06:03 -0400 Subject: [PATCH 27/34] fix lora load diffusers --- CHANGELOG.md | 2 ++ extensions-builtin/Lora/network.py | 3 ++- extensions-builtin/Lora/network_overrides.py | 14 +++++++++----- extensions-builtin/Lora/networks.py | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ed274e84..4e5263104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,8 @@ ### Improvements +- further work on improving python 3.12 functionality and remove experimental flag + note: recommended version remains python 3.11 for all users except if you're using directml and then its python 3.10 - improved **installer** for initial installs initial install will do single-pass install of all required packages with correct versions subsequent runs will check package versions as necessary diff --git a/extensions-builtin/Lora/network.py b/extensions-builtin/Lora/network.py index 9558047d9..a6579ae90 100644 --- a/extensions-builtin/Lora/network.py +++ b/extensions-builtin/Lora/network.py @@ -31,7 +31,8 @@ class NetworkOnDisk: self.metadata = m self.alias = self.metadata.get('ss_output_name', self.name) # self.set_hash(self.metadata.get('sshs_model_hash') or hashes.sha256_from_cache(self.filename, "lora/" + self.name, use_addnet_hash=self.is_safetensors) or '') - self.set_hash(hashes.sha256_from_cache(self.filename, "lora/" + self.name) or self.metadata.get('sshs_model_hash')) + sha256 = hashes.sha256_from_cache(self.filename, "lora/" + self.name) or hashes.sha256_from_cache(self.filename, "lora/" + self.name, use_addnet_hash=True) or self.metadata.get('sshs_model_hash') + self.set_hash(sha256) self.sd_version = self.detect_version() def detect_version(self): diff --git a/extensions-builtin/Lora/network_overrides.py b/extensions-builtin/Lora/network_overrides.py index a7f56327f..724e47c70 100644 --- a/extensions-builtin/Lora/network_overrides.py +++ b/extensions-builtin/Lora/network_overrides.py @@ -1,7 +1,7 @@ from modules import shared -force_diffusers = [ +maybe_diffusers = [ 'aaebf6360f7d', # sd15-lcm '3d18b05e4f56', # sdxl-lcm 'b71dcb732467', # sdxl-tcd @@ -19,12 +19,16 @@ force_diffusers = [ '8cca3706050b', # hyper-sdxl-1step ] +force_diffusers = [ + '816d0eed49fd', # flash-sdxl + 'c2ec22757b46', # flash-sd15 +] + def check_override(shorthash): - if not shared.opts.lora_maybe_diffusers: - return False if len(shorthash) < 4: return False - force = any(x.startswith(shorthash) for x in force_diffusers) - if force: + force = any(x.startswith(shorthash) for x in maybe_diffusers) if shared.opts.lora_maybe_diffusers else False + force = force or any(x.startswith(shorthash) for x in force_diffusers) + if force and shared.opts.lora_maybe_diffusers: shared.log.debug('LoRA override: force diffusers') return force diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index d36809d47..e564e2b67 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -85,7 +85,7 @@ def load_diffusers(name, network_on_disk, lora_scale=1.0) -> network.Network: shared.log.debug(f'LoRA load: name="{name}" file="{network_on_disk.filename}" type=diffusers {"cached" if cached else ""} fuse={shared.opts.lora_fuse_diffusers}') if cached is not None: return cached - if shared.native: + if not shared.native: return None shared.sd_model.load_lora_weights(network_on_disk.filename) if shared.opts.lora_fuse_diffusers: From 4840c0019f5fa030a6f185108a3bddac8c41f807 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 8 Jun 2024 15:17:07 -0400 Subject: [PATCH 28/34] cleanup installer --- installer.py | 3 ++- modules/loader.py | 6 ------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/installer.py b/installer.py index 9c77a1cf2..a07444abf 100644 --- a/installer.py +++ b/installer.py @@ -761,7 +761,8 @@ def install_packages(): clip_package = os.environ.get('CLIP_PACKAGE', "git+https://github.com/openai/CLIP.git") install(clip_package, 'clip', quiet=True) tensorflow_package = os.environ.get('TENSORFLOW_PACKAGE', 'tensorflow==2.13.0') - install(tensorflow_package, 'tensorflow-rocm' if 'rocm' in tensorflow_package else 'tensorflow', ignore=True, quiet=True) + if tensorflow_package != '': + install(tensorflow_package, 'tensorflow-rocm' if 'rocm' in tensorflow_package else 'tensorflow', ignore=True, quiet=True) bitsandbytes_package = os.environ.get('BITSANDBYTES_PACKAGE', None) if bitsandbytes_package is not None: install(bitsandbytes_package, 'bitsandbytes', ignore=True, quiet=True) diff --git a/modules/loader.py b/modules/loader.py index 04e6dfca2..1fe3ba81e 100644 --- a/modules/loader.py +++ b/modules/loader.py @@ -10,14 +10,8 @@ from modules import timer, errors initialized = False errors.install() logging.getLogger("DeepSpeed").disabled = True -# os.environ.setdefault('OMP_NUM_THREADS', 1) -# os.environ.setdefault('MKL_NUM_THREADS', 1) - -# import tensorflow as tf # pylint: disable=C0411 import torch # pylint: disable=C0411 - -# torch.set_num_threads(1) try: import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import errors.log.debug(f'Load IPEX=={ipex.__version__}') From 0fa68f5a9bf1a78e3ec518d876e36f85a649d027 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 8 Jun 2024 16:08:56 -0400 Subject: [PATCH 29/34] no not install tensorflow by default --- CHANGELOG.md | 1 + installer.py | 27 ++++++++++++++------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e5263104..1bdb6750f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ - add env variable `SD_PIP_DEBUG` to write `pip.log` for all pip operations also improved installer logging - add python version check for `torch-directml` +- do not install `tensorflow` by default - improve metadata/infotext parser add `cli/image-exif.py` that can be used to view/extract metadata from images - lower overhead on generate calls diff --git a/installer.py b/installer.py index a07444abf..686dd6c02 100644 --- a/installer.py +++ b/installer.py @@ -438,8 +438,8 @@ def install_rocm_zluda(torch_command): is_windows = platform.system() == 'Windows' log.info('AMD ROCm toolkit detected') os.environ.setdefault('PYTORCH_HIP_ALLOC_CONF', 'garbage_collection_threshold:0.8,max_split_size_mb:512') - if not is_windows: - os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow-rocm') + # if not is_windows: + # os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow-rocm') try: if is_windows: command = subprocess.run('hipinfo', shell=True, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -468,8 +468,8 @@ def install_rocm_zluda(torch_command): os.environ.setdefault('HIP_VISIBLE_DEVICES', str(idx)) if arch == 'navi3x': os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', '11.0.0') - if os.environ.get('TENSORFLOW_PACKAGE') == 'tensorflow-rocm': # do not use tensorflow-rocm for navi 3x - os.environ['TENSORFLOW_PACKAGE'] = 'tensorflow==2.13.0' + # if os.environ.get('TENSORFLOW_PACKAGE') == 'tensorflow-rocm': # do not use tensorflow-rocm for navi 3x + # os.environ['TENSORFLOW_PACKAGE'] = 'tensorflow==2.13.0' elif arch == 'navi2x': os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', '10.3.0') else: @@ -551,7 +551,7 @@ def install_ipex(torch_command): os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100') if "linux" in sys.platform: torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.1.0.post0 torchvision==0.16.0.post0 intel-extension-for-pytorch==2.1.20+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/') - os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow==2.15.0 intel-extension-for-tensorflow[xpu]==2.15.0.0') + # os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow==2.15.0 intel-extension-for-tensorflow[xpu]==2.15.0.0') if os.environ.get('DISABLE_VENV_LIBS', None) is None: install(os.environ.get('MKL_PACKAGE', 'mkl==2024.1.0'), 'mkl') install(os.environ.get('DPCPP_PACKAGE', 'mkl-dpcpp==2024.1.0'), 'mkl-dpcpp') @@ -760,14 +760,15 @@ def install_packages(): log.info('Verifying packages') clip_package = os.environ.get('CLIP_PACKAGE', "git+https://github.com/openai/CLIP.git") install(clip_package, 'clip', quiet=True) - tensorflow_package = os.environ.get('TENSORFLOW_PACKAGE', 'tensorflow==2.13.0') - if tensorflow_package != '': - install(tensorflow_package, 'tensorflow-rocm' if 'rocm' in tensorflow_package else 'tensorflow', ignore=True, quiet=True) - bitsandbytes_package = os.environ.get('BITSANDBYTES_PACKAGE', None) - if bitsandbytes_package is not None: - install(bitsandbytes_package, 'bitsandbytes', ignore=True, quiet=True) - elif not args.experimental: - uninstall('bitsandbytes') + # tensorflow_package = os.environ.get('TENSORFLOW_PACKAGE', 'tensorflow==2.13.0') + # tensorflow_package = os.environ.get('TENSORFLOW_PACKAGE', None) + # if tensorflow_package is not None: + # install(tensorflow_package, 'tensorflow-rocm' if 'rocm' in tensorflow_package else 'tensorflow', ignore=True, quiet=True) + # bitsandbytes_package = os.environ.get('BITSANDBYTES_PACKAGE', None) + # if bitsandbytes_package is not None: + # install(bitsandbytes_package, 'bitsandbytes', ignore=True, quiet=True) + # elif not args.experimental: + # uninstall('bitsandbytes') if args.profile: print_profile(pr, 'Packages') From 7f984c66fa0b891682a4fb2365549c3a1c74fae8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 9 Jun 2024 08:15:34 -0400 Subject: [PATCH 30/34] fix control reference enabled check --- modules/control/run.py | 12 +++++++----- modules/hidiffusion/hidiffusion.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/modules/control/run.py b/modules/control/run.py index 1357be713..aa606e66a 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -254,6 +254,8 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini control_conditioning = active_strength[0] if len(active_strength) == 1 else list(active_strength) # strength or list[strength] control_guidance_start = active_start[0] if len(active_start) == 1 else list(active_start) control_guidance_end = active_end[0] if len(active_end) == 1 else list(active_end) + elif unit_type == 'reference': + has_models = any(u.enabled for u in units if u.type == 'reference') else: pass @@ -299,7 +301,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini pipe = instance.pipeline if inits is not None: shared.log.warning('Control: ControlLLLite does not support separate init image') - elif unit_type == 'reference': + elif unit_type == 'reference' and has_models: p.extra_generation_params["Control mode"] = 'Reference' p.extra_generation_params["Control attention"] = p.attention p.task_args['reference_attn'] = 'Attention' in p.attention @@ -488,7 +490,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini debug('Control processed: using input direct') processed_image = input_image - if unit_type == 'reference': + if unit_type == 'reference' and has_models: p.ref_image = p.override or input_image p.task_args.pop('image', None) p.task_args['ref_image'] = p.ref_image @@ -496,11 +498,11 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini if p.ref_image is None: yield terminate('Control: attempting reference mode but image is none') return [], '', '', 'Reference mode without image' - elif unit_type == 'controlnet' and input_type == 1: # Init image same as control + elif unit_type == 'controlnet' and input_type == 1 and has_models: # Init image same as control p.task_args['control_image'] = p.init_images # switch image and control_image p.task_args['strength'] = p.denoising_strength p.init_images = [p.override or input_image] * len(active_model) - elif unit_type == 'controlnet' and input_type == 2: # Separate init image + elif unit_type == 'controlnet' and input_type == 2 and has_models: # Separate init image if init_image is None: shared.log.warning('Control: separate init image not provided') init_image = input_image @@ -517,7 +519,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini t2 += time.time() - t2 # determine txt2img, img2img, inpaint pipeline - if unit_type == 'reference': # special case + if unit_type == 'reference' and has_models: # special case p.is_control = True shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) elif not has_models: # run in txt2img/img2img/inpaint mode diff --git a/modules/hidiffusion/hidiffusion.py b/modules/hidiffusion/hidiffusion.py index dc0f8d991..df866bce8 100644 --- a/modules/hidiffusion/hidiffusion.py +++ b/modules/hidiffusion/hidiffusion.py @@ -228,7 +228,7 @@ def make_diffusers_transformer_block(block_class: Type[torch.nn.Module]) -> Type norm_hidden_states = self.norm2(hidden_states) norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp if self._chunk_size is not None: - ff_output = _chunked_feed_forward(self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size) # TODO hidiffusion undefined + ff_output = _chunked_feed_forward(self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size) # pylint: disable=undefined-variable # TODO hidiffusion undefined else: ff_output = self.ff(norm_hidden_states) if self.use_ada_layer_norm_zero: From d1ae428f4bff9ff58d232a18fc2744586e0f1c60 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 9 Jun 2024 08:32:31 -0400 Subject: [PATCH 31/34] fix facehires with control batch count --- CHANGELOG.md | 11 ++++++++--- scripts/face-details.py | 4 ++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bdb6750f..82704c6e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,14 +59,19 @@ subsequent runs will check package versions as necessary - add env variable `SD_PIP_DEBUG` to write `pip.log` for all pip operations also improved installer logging -- add python version check for `torch-directml` +- add python version check for `torch-directml` - do not install `tensorflow` by default - improve metadata/infotext parser add `cli/image-exif.py` that can be used to view/extract metadata from images -- lower overhead on generate calls +- lower overhead on generate calls - auto-synchronize modernui and core branches -- fix apply/unapply hidiffusion for sd15 + +## Fixes + - cumulative fixes since the last release +- fix apply/unapply hidiffusion for sd15 +- fix controlnet reference enabled check +- fix face-hires with control batch count ## Update for 2024-06-02 diff --git a/scripts/face-details.py b/scripts/face-details.py index 2a53b4234..3fd77fe69 100644 --- a/scripts/face-details.py +++ b/scripts/face-details.py @@ -137,8 +137,10 @@ class FaceRestorerYolo(FaceRestoration): 'width': resolution, 'height': resolution, } + control_pipeline = None if getattr(p, 'is_control', False): from modules.control import run + control_pipeline = shared.sd_model run.restore_pipeline() p = processing_class.switch_class(p, processing.StableDiffusionProcessingImg2Img, args) @@ -171,6 +173,8 @@ class FaceRestorerYolo(FaceRestoration): mask_all.append(pp.images[1]) # restore pipeline + if control_pipeline is not None: + shared.sd_model = control_pipeline p = processing_class.switch_class(p, orig_cls, orig_p) p.init_images = getattr(orig_p, 'init_images', None) p.image_mask = getattr(orig_p, 'image_mask', None) From 773a5087ccfabc9090fe0991a8d0776e0150ba1e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 9 Jun 2024 09:25:52 -0400 Subject: [PATCH 32/34] control add before after sizing to metadata --- cli/image-exif.py | 2 +- modules/control/run.py | 24 +++++++++++++++++++++++- modules/processing_class.py | 18 ++++++++++++++++++ modules/processing_info.py | 15 +++++++++++++++ modules/ui_control.py | 18 +++++++++++++++++- 5 files changed, 74 insertions(+), 3 deletions(-) diff --git a/cli/image-exif.py b/cli/image-exif.py index 0aa7ffc55..c3954204a 100755 --- a/cli/image-exif.py +++ b/cli/image-exif.py @@ -4,7 +4,7 @@ import os import io import re import sys -import importlib +import importlib.util from PIL import Image, ExifTags, TiffImagePlugin, PngImagePlugin from rich import print # pylint: disable=redefined-builtin diff --git a/modules/control/run.py b/modules/control/run.py index aa606e66a..5a41b87e4 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -149,12 +149,34 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini shared.log.debug('Control: override resize mode=mask') selected_scale_tab_mask = 1 - # set initial resolution + # set control sizing if resize_mode_before != 0 or inputs is None or inputs == [None]: p.width, p.height = width_before, height_before # pylint: disable=attribute-defined-outside-init + p.width_before = width_before + p.height_before = height_before + if resize_name_before != 'None': + p.resize_mode_before = resize_mode_before + p.resize_name_before = resize_name_before + p.scale_by_before = scale_by_before + p.selected_scale_tab_before = selected_scale_tab_before else: del p.width del p.height + if resize_name_after != 'None': + p.resize_mode_after = resize_mode_after + p.resize_name_after = resize_name_after + p.width_after = width_after + p.height_after = height_after + p.scale_by_after = scale_by_after + p.selected_scale_tab_after = selected_scale_tab_after + if resize_name_mask != 'None': + p.resize_mode_mask = resize_mode_mask + p.resize_name_mask = resize_name_mask + p.width_mask = width_mask + p.height_mask = height_mask + p.scale_by_mask = scale_by_mask + p.selected_scale_tab_mask = selected_scale_tab_mask + # hires/refine defined outside of main init p.enable_hr = enable_hr p.hr_sampler_name = processing.get_sampler_name(hr_sampler_index) diff --git a/modules/processing_class.py b/modules/processing_class.py index 9f8ac7792..bf37c8e2d 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -469,6 +469,24 @@ class StableDiffusionProcessingControl(StableDiffusionProcessingImg2Img): self.fidelity = 0.5 self.mask_image = None self.override = None + self.resize_mode_before = None + self.resize_name_before = None + self.width_before = None + self.height_before = None + self.scale_by_before = None + self.selected_scale_tab_before = None + self.resize_mode_after = None + self.resize_name_after = None + self.width_after = None + self.height_after = None + self.scale_by_after = None + self.selected_scale_tab_after = None + self.resize_mode_mask = None + self.resize_name_mask = None + self.width_mask = None + self.height_mask = None + self.scale_by_mask = None + self.selected_scale_tab_mask = None def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): # abstract pass diff --git a/modules/processing_info.py b/modules/processing_info.py index fb376627f..c6f572dae 100644 --- a/modules/processing_info.py +++ b/modules/processing_info.py @@ -98,6 +98,21 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No # lookup by index if getattr(p, 'resize_mode', None) is not None: args['Resize mode'] = shared.resize_modes[p.resize_mode] if shared.resize_modes[p.resize_mode] != 'None' else None + if getattr(p, 'resize_mode_before', None) is not None: + args['Size before'] = f"{p.width_before}x{p.height_before}" if hasattr(p, 'width_before') and hasattr(p, 'height_before') else None + args['Size mode before'] = p.resize_mode_before + args['Size scale before'] = p.scale_by_before + args['Size name before'] = p.resize_name_before + if getattr(p, 'resize_mode_after', None) is not None: + args['Size after'] = f"{p.width_after}x{p.height_after}" if hasattr(p, 'width_after') and hasattr(p, 'height_after') else None + args['Size mode after'] = p.resize_mode_after + args['Size scale after'] = p.scale_by_after + args['Size name after'] = p.resize_name_after + if getattr(p, 'resize_mode_mask', None) is not None: + args['Size mask'] = f"{p.width_mask}x{p.height_mask}" if hasattr(p, 'width_mask') and hasattr(p, 'height_mask') else None + args['Size mode mask'] = p.resize_mode_mask + args['Size scale mask'] = p.scale_by_mask + args['Size name mask'] = p.resize_name_mask if 'face' in p.ops: args["Face restoration"] = shared.opts.face_restoration_model if 'color' in p.ops: diff --git a/modules/ui_control.py b/modules/ui_control.py index 1bf378656..c4aa25bd3 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -533,11 +533,27 @@ def create_ui(_blocks: gr.Blocks=None): (negative, "Negative prompt"), # input (denoising_strength, "Denoising strength"), - # resize + # size basic (width_before, "Size-1"), (height_before, "Size-2"), (resize_mode_before, "Resize mode"), (scale_by_before, "Resize scale"), + # size control + (width_before, "Size before-1"), + (height_before, "Size before-2"), + (resize_mode_before, "Size mode before"), + (scale_by_before, "Size scale before"), + (resize_name_before, "Size name before"), + (width_after, "Size after-1"), + (height_after, "Size after-2"), + (resize_mode_after, "Size mode after"), + (scale_by_after, "Size scale after"), + (resize_name_after, "Size name after"), + (width_mask, "Size mask-1"), + (height_mask, "Size mask-2"), + (resize_mode_mask, "Size mode mask"), + (scale_by_mask, "Size scale mask"), + (resize_name_mask, "Size name mask"), # sampler (sampler_index, "Sampler"), (steps, "Steps"), From 8a541d21fa3513f65a279bbbb3965e665b4f6e41 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 9 Jun 2024 09:54:16 -0400 Subject: [PATCH 33/34] prep work for single-file t2i-adapters --- modules/control/units/t2iadapter.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/modules/control/units/t2iadapter.py b/modules/control/units/t2iadapter.py index fda33ec0b..1c481398b 100644 --- a/modules/control/units/t2iadapter.py +++ b/modules/control/units/t2iadapter.py @@ -22,6 +22,12 @@ predefined_sd15 = { 'Canny v2': 'TencentARC/t2iadapter_canny_sd15v2', 'Sketch v1': 'TencentARC/t2iadapter_sketch_sd14v1', 'Sketch v2': 'TencentARC/t2iadapter_sketch_sd15v2', + # 'Coadapter Canny': 'TencentARC/T2I-Adapter/models/coadapter-canny-sd15v1.pth', + # 'Coadapter Color': 'TencentARC/T2I-Adapter/models/coadapter-color-sd15v1.pth', + # 'Coadapter Depth': 'TencentARC/T2I-Adapter/models/coadapter-depth-sd15v1.pth', + # 'Coadapter Fuser': 'TencentARC/T2I-Adapter/models/coadapter-fuser-sd15v1.pth', + # 'Coadapter Sketch': 'TencentARC/T2I-Adapter/models/coadapter-sketch-sd15v1.pth', + # 'Coadapter Style': 'TencentARC/T2I-Adapter/models/coadapter-style-sd15v1.pth', } predefined_sdxl = { 'Canny XL': 'TencentARC/t2i-adapter-canny-sdxl-1.0', @@ -31,6 +37,7 @@ predefined_sdxl = { 'OpenPose XL': 'TencentARC/t2i-adapter-openpose-sdxl-1.0', 'Midas Depth XL': 'TencentARC/t2i-adapter-depth-midas-sdxl-1.0', } + models = {} all_models = {} all_models.update(predefined_sd15) @@ -94,7 +101,15 @@ class Adapter(): log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id') return log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}"') - self.model = T2IAdapter.from_pretrained(model_path, **self.load_config) + if model_path.endswith('.pth') or model_path.endswith('.pt') or model_path.endswith('.safetensors'): + from huggingface_hub import hf_hub_download + parts = model_path.split('/') + repo_id = f'{parts[0]}/{parts[1]}' + filename = '/'.join(parts[2:]) + model = hf_hub_download(repo_id, filename, **self.load_config) + self.model = T2IAdapter.from_pretrained(model, **self.load_config) + else: + self.model = T2IAdapter.from_pretrained(model_path, **self.load_config) if self.device is not None: self.model.to(self.device) if self.dtype is not None: From 3341a34549ef310454f40bd15728310058672cd2 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 9 Jun 2024 11:15:31 -0400 Subject: [PATCH 34/34] cuda exception handler --- modules/zluda.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/zluda.py b/modules/zluda.py index 033b7c93a..e244c245d 100644 --- a/modules/zluda.py +++ b/modules/zluda.py @@ -18,8 +18,11 @@ def _join_rocm_home(*paths) -> str: def is_zluda(device: DeviceLikeType): - device = torch.device(device) - return torch.cuda.get_device_name(device).endswith("[ZLUDA]") + try: + device = torch.device(device) + return torch.cuda.get_device_name(device).endswith("[ZLUDA]") + except Exception: + return False def test(device: DeviceLikeType) -> Union[Exception, None]: