From e32220ccc1e46b0a22f4dfb20239824aa8893bf1 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 5 Feb 2024 10:38:42 -0500 Subject: [PATCH] experimental segmoe support --- CHANGELOG.md | 26 +- extensions-builtin/Lora/network_lora.py | 2 +- html/reference.json | 15 + models/Reference/segmind--SegMoE-2x1-v0.jpg | Bin 0 -> 21996 bytes models/Reference/segmind--SegMoE-4x2-v0.jpg | Bin 0 -> 21996 bytes .../Reference/segmind--SegMoE-SD-4x2-v0.jpg | Bin 0 -> 21996 bytes .../control/proc/marigold/util/ensemble.py | 4 +- modules/face/instantid_model.py | 2 +- modules/processing_diffusers.py | 11 +- modules/sd_models.py | 12 + modules/segmoe/segmoe_model.py | 1315 +++++++++++++++++ modules/shared_items.py | 3 +- pyproject.toml | 1 + 13 files changed, 1376 insertions(+), 15 deletions(-) create mode 100644 models/Reference/segmind--SegMoE-2x1-v0.jpg create mode 100644 models/Reference/segmind--SegMoE-4x2-v0.jpg create mode 100644 models/Reference/segmind--SegMoE-SD-4x2-v0.jpg create mode 100644 modules/segmoe/segmoe_model.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5590b2ee6..c31c4daa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,14 @@ # Change Log for SD.Next -## Future +## TODO Future - ipadapter multi image - control second pass - diffusers public callbacks - image2video: pia and vgen pipelines +- video2video - wuerstchen v3 [pr](https://github.com/huggingface/diffusers/pull/6487) - more pipelines: -- segmoe: - control api - masking api - preprocess api @@ -21,12 +21,12 @@ - update docs - diffusers 0.26.2 -## Update for 2023-02-04 +## TODO Release notes Another big release, highlights being: - A lot more functionality in the **Control** module: - Inpaint and outpaint support, flexible resizing options, optional hires - - Built-in support for many new processors and models which are auto-downloaded on first use + - Built-in support for many new processors and models, all auto-downloaded on first use - Full support for scripts and extensions - Complete **Face** module implements all variations of **FaceID**, **FaceSwap** and latest **PhotoMaker** and **InstantID** @@ -34,14 +34,21 @@ Another big release, highlights being: - Brand new **Intelligent masking**, manual or automatic Using ML models (*LAMA* object removal, *REMBG* background removal, *SAM* segmentation, etc.) and with live previews With granular blur, erode and dilate controls +- New models and pipelines: + **Segmind SegMoE**, **Mixture Tiling**, **InstaFlow**, **SAG**, **BlipDiffusion** - Massive work integrating latest advances with [OpenVINO](https://github.com/vladmandic/automatic/wiki/OpenVINO), [IPEX](https://github.com/vladmandic/automatic/wiki/Intel-ARC) and [ONNX Olive](https://github.com/vladmandic/automatic/wiki/ONNX-Runtime-&-Olive) -- **New models** and pipelines: *Mixture Tiling*, *SAG*, *InstaFlow*, *BlipDiffusion* - Full control over brightness, sharpness and color during generate process directly in latent space Plus welcome additions to **UI performance, usability and accessibility** and flexibility of deployment And it also includes fixes for all reported issues so far -As of this release, default backend is set to **diffusers** as its more feature rich than **original** and supports many additional models +As of this release, default backend is set to **diffusers** as its more feature rich than **original** and supports many additional models (original backend does remain as fully supported) + +- For basic instructions, see [README](https://github.com/vladmandic/automatic/blob/master/README.md) +- For more details on all new features see full [CHANGELOG](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) +- For documentation, see [WIKI](https://github.com/vladmandic/automatic/wiki) + +## Update for 2023-02-04 - **Control**: - add **inpaint** support @@ -129,6 +136,13 @@ As of this release, default backend is set to **diffusers** as its more feature **SD15**: Base, Base ViT-G, Light, Plus, Plus Face, Full Face **SDXL**: Base SXDL, Base ViT-H SXDL, Plus ViT-H SXDL, Plus Face ViT-H SXDL - enable use via api, thanks @trojaner +- [Segmind SegMoE](https://github.com/segmind/segmoe) + - initial support for reference models + download&load via network -> models -> reference -> **SegMoE SD 4x2** (3.7GB), **SegMoE XL 2x1** (10GB), **SegMoE XL 4x2** + - note: since segmoe is basically sequential mix of unets from multiple models, it can get large + SD 4x2 is ~4GB, XL 2x1 is ~10GB and XL 4x2 is 18GB + - support for create and load custom mixes will be added in the future + - support for lora and other advanced features will be added in the future - [Mixture Tiling](https://arxiv.org/abs/2302.02412) - uses multiple prompts to guide different parts of the grid during diffusion process - can be used ot create complex scenes with multiple subjects diff --git a/extensions-builtin/Lora/network_lora.py b/extensions-builtin/Lora/network_lora.py index 8f31388d4..1405a9d13 100644 --- a/extensions-builtin/Lora/network_lora.py +++ b/extensions-builtin/Lora/network_lora.py @@ -42,7 +42,7 @@ class NetworkModuleLora(network.NetworkModule): elif is_conv and key == "lora_up.weight" or key == "dyn_down": module = torch.nn.Conv2d(weight.shape[1], weight.shape[0], (1, 1), bias=False) else: - raise AssertionError(f'Lora layer {self.network_key} matched a layer with unsupported type: {type(self.sd_module).__name__}') + raise AssertionError(f'Lora unsupported: layer={self.network_key} type={type(self.sd_module).__name__}') with torch.no_grad(): if weight.shape != module.weight.shape: weight = weight.reshape(module.weight.shape) diff --git a/html/reference.json b/html/reference.json index eadcec5be..0d48447fd 100644 --- a/html/reference.json +++ b/html/reference.json @@ -88,6 +88,21 @@ "desc": "Segmind's Tiny-SD offers a compact, efficient, and distilled version of Realistic Vision 4.0 and is up to 80% faster than SD1.5", "preview": "segmind--tiny-sd.jpg" }, + "Segmind SegMoE SD 4x2": { + "path": "segmind/SegMoE-SD-4x2-v0", + "desc": "SegMoE-SD-4x2-v0 is an untrained Segmind Mixture of Diffusion Experts Model generated using segmoe from 4 Expert SD1.5 models. SegMoE is a powerful framework for dynamically combining Stable Diffusion Models into a Mixture of Experts within minutes without training", + "preview": "segmind--SegMoE-SD-4x2-v0.jpg" + }, + "Segmind SegMoE XL 2x1": { + "path": "segmind/SegMoE-2x1-v0", + "desc": "SegMoE-2x1-v0 is an untrained Segmind Mixture of Diffusion Experts Model generated using segmoe from 2 Expert SDXL models. SegMoE is a powerful framework for dynamically combining Stable Diffusion Models into a Mixture of Experts within minutes without training", + "preview": "segmind--SegMoE-2x1-v0.jpg" + }, + "Segmind SegMoE XL 4x2": { + "path": "segmind/SegMoE-4x2-v0", + "desc": "SegMoE-4x2-v0 is an untrained Segmind Mixture of Diffusion Experts Model generated using segmoe from 4 Expert SDXL models. SegMoE is a powerful framework for dynamically combining Stable Diffusion Models into a Mixture of Experts within minutes without training", + "preview": "segmind--SegMoE-4x2-v0.jpg" + }, "LCM SD-1.5 Dreamshaper 7": { "path": "SimianLuo/LCM_Dreamshaper_v7", "desc": "Latent Consistencey Models enable swift inference with minimal steps on any pre-trained LDMs, including Stable Diffusion. By distilling classifier-free guidance into the model's input, LCM can generate high-quality images in very short inference time. LCM can generate quality images in as few as 3-4 steps, making it blazingly fast.", diff --git a/models/Reference/segmind--SegMoE-2x1-v0.jpg b/models/Reference/segmind--SegMoE-2x1-v0.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7d7989ec2a1c0825b092263fe09324ed9a995119 GIT binary patch literal 21996 zcmbTdc|4Tg|37@qFvw0r_MOI>Wyo&qL)Njx&|+tjvQ|nBvNYDQ#Uu^JFt#YW6xr7- zLy}6eE4>RzO25nd^Z9=7$K&_cecu=7T<2VKjO+Ehp68s`cKx0Gy8v)oTUc2D5C{ag z2>t+nKLWny*Ms~4z}6N}004jmV1x((Ft7yyiy=h#f7=!i1ifbcjsZUh7V|{%6`~w1maN!Y=QP-~D zh>pLTkeHNwkC2s}lbd%xzkpOyN-3k3SI{b-JZ)%fYHn$5d)eL7+t)uZI5as$pPqR= z`)2Or;-{tMmDRQNjc?z#cXog5{oMa`Bo_pf^M8*22=;%;#RJL(Wn^SvWImD$0=)(n z1|CM3q#6^iu_Lor=y56aSQfYmvADkLn6w7wA3pD}308g?JtKXT0h>Q;^of>?6Ivc3MKUKJbDm^nvZfMkf&W${4#{kJDvf+B zXUh1fa-I~_C3%y8hqh|AfSZ#ejm*YM??vJtVoRM+DUeE4I=)V*gab7Usg$6z)CrYx zppGY2bUR30o#w_}wEMTuR1xpM__or3&T?Oqe?x__wW8Vas_Z+*kdg&7wnQ1#pI4FM zz>wnXM*|k8d2mefxYR~KQ7E-hGG5bS6vZHvM#3{`3Tvvx3wa@d)S%Qrh`^M# z+{Q$WnvLS{LEEYd^^y8w!4@6pqaZtj$m0wgOtI`pApmSb%imM%J5c=h%h)C<#dHU? z8e=lcTq0HKL>bOE(?D^WC4q*`Pac)0N@*CAnI=%I2zVm8L0K}ri~_}s1M)Cc8F(CC zhAGp55kJXlD(i&i01+c`05((-6))>#>xB00JVMO*Kd`~a)g!^1fczIb3l0e-2FFD5 zv2JIt3h32o0UYW-&?JJ;!0;ecK%G!A#}S|sbci%_NFZc)r>v;Z z{7DW+rhjpyK~!rL~voSlU6s6-IMVhwZ z<=#1Lk&iVy>s3H8)5mZ?W}^uYP3Sc?X8t+jjF+wHEUDu`_>x$(&rB}PLK{P1 zl2w+Fg0q%bvKpfXVG`tH|0&2sG=~Hd@*fbVj46!dBW@W}<3TvViZ|YdDK;zKmI*J* zMBGS;|a&w;U>=AFmx9hmMEarPZ0rJWa++ zJi-D|8!yDt^hWD%Z)vo{v5RepkeaEFi{pr~*P=1&-|NFm+5OtsUpp}+u)FE#^dt8L zS|^5hEwP&?%$hluQnz+XiAlOTN~HC~SRq9qJiAxLRb*}D^jW*-Vt0p9-!VIG+UJ4; zCa%akytO-4JEJGy<4|k?GqgArm~OuF<_yAIf46q9#9XkmG*f5n^nuEid_Emf#Ttd! z`dI6(k*gVuRVVWtw#F2j4)m4YHl-4j41EY~4O#E_9&|*h&vt1rNX0SROk8a3_k=11 zcZvL7{R@cIb~L=bA!ud%J>~-r7M|A76tp^P46u_vPuV7_j}T4^DT(y+&U{@NfZwV> zk4Ao9TbJO3ktipYKJt9-5>utLsd= zMzNCX4FJA1eynYJ{)={-R+08FTfJ*1AJo6J^b1$aNk1l-5MN;#qn?J(C$ycgLR6x` z-CmOh;wQbA<9Y>snUfF>c&Diq3MLCE?2;oX>YvXclFH7=A9s*vH?~hAwbz%ycp)`^NnbAFm%Ri>RY5tcyxO86goe|7f7tNX?uDH+8qt-wo{c zA=q-m&;AAKHQDdg6 zTdAHavPE&Ofy*M0*ZQx`bVOVbQa9dU6s+5L0?Bw4_o~r{mjm~uH2Z{1kdhe6(pfF+dbS~4ElAyzf8sJJ4rlHU_*DtX7bC%b^WEN zvXTCw;Oe1E$7+ZAOdfVljEIz@SPV85`gVrz8d%n7N`{f%R8F2#h#fWba=Vqq@zFLw zb1}J~?WqXsgY@p&Et|kCrmI(h4O~tbQuSG?DSE^@{*UM7%WlEJjSvL{1Y+@+kBxyBt)17MJ6hBjU z?X=Yi_JkHbc3V=`XFE>4$;w#zxRhHVZJ^9~6w|8Gn_BZyV4j`m=qtHb18*-FY{>9{yv$&zI3o)u8pvY-Ct%OMI6SQL?m*|=v5fibA+C=-! z>awDBou3eW|HDk^(Sj+3bER4_Om<*+*f8LNc|=TGYdS=bJPv?bjXpT?IvE3$;1vkQ zuIP^_3PS=kHUrddKyFI@_YCM`Sb^&xFRvrP>l{c2wS`IW0&)Vy%SsxXyfk>Lg}e38 z8KRkxy0^{p(eqdRnqt$GlGl_{CtH&Aiosxc$kKL{)D%+hCgTUm50V@9J2VxQGsm+r zt9*5InLTBza1kF?n9muVFQfjOhKYpOsvi?y+~k&X=m$Y3KJ!c{ztrwVNY(Kkh}vqp zXRV1cUVY0nPaywtmAa6pXS~wja%DWpytdrZUG+TrnRbnHt@lSC+a;Gx_opN7*O6Ms z`2E_Pg`f7{33bc3I`Bbl|Bk-%e65p-$J%V|V2=2YSA4NITQA6P)VvbqY9U9R_Uqs| z{q@9EqrN1*;kfQ{DSZ92wk+#wHy-nAOQf#Uk7xIV5>Y2=Wdkx*vrcyn_zC>p$YOug zv5$dnj!jlD!VF70%=U$9;I(`o4^JqYMXdHsbn%4IL+xpQBYoK!mI31?5X(xl8Y|+b4-Kp^PS`lfE9<%#AsrE#Y zC~R0liJH@(y9Q5dIcKS*rgLa5IymQQYi4VnMVwzaRa|+tS&&h!7#DPj=SB}t`_?Zy zdgz7|aqUv%@nbeq$-vM^Sb57AN9+Lf%lhX_X2g_i%}LCO+KCO0@ z*T0;LGxole_-?Y~z&9-8eNXux%D(+i2MGLAP~2@F1Fi_elKerDq&@YJzkpw&@L%Au z4a;);U}jwCJkNNBqm#w8Q^ZrlfBKAG-gz0zU5f7}_Sa~cUtdzNm=QGx+&CJxn>Q4= zG|CM+0tGMoN$iMwy7&&CX2PG6Nu5t*o^=!w(?~7a2Q}-uF6s}7KL}_ zMs&1U4LgF8g9NpAHZhmy4D7c{y2A7?`jVo$r`!Gl3Q8xqeGf2`0uXamo<8$aSd?q7 zdby^KBYj*ie@`trv+!15&3u2mk|i|Tyw5^mVCaJmb)I20M4xrAXCt*1d%TIc48=8^ zJz1VZ9ZeH6S+V#Gv!tj$(0LTagj=i*ceysc-1sAT1H;l5zbzj9xKjUAR3B%-(ywW0 z^)gwt6ssc>58oVeozlFH88WX&Q5-g`K~r&xyRVp*e6Ipmf+m6Xp@uQ_NC^d!CrN-Y zilVj%s+Yac%+5&?$`Q^w9=pcI4fCwSw}(g-5B181jg~b5i;b5T)_Z-I0!wt~*x45w zE+(bto)%4Z75PyEkDp74MR^>!=Eu)v3N9)|)iA)+1YYy+zv3%b(vK($E$n#rm@(D^ z#70aZC&4FAg4R`af2LRXDqm&pAg)(6NIp4I7Ai)~6au#N<^BR|*SGgFwjO<~zQT5{ z7xu=AA>{Vmh~hxG^tjzgJu2g3@oRiqR`Av9nmIGn9TKi{JH(Tz`kvhWHTh}}UHCDT z;XZX?O^cxOW=5Kd{JE(I?eWr|B!p;2yoHC6&!kc}tV&H_Dg~mJp)QvT#^W-G*uomp zgiUCZ^U@veikVMWyEy|To=h!7E>@g539g!QSq}X*4(j1uNS2nzhvStK{66C`fza=& zuNZ}bgHf9{Y~PsgFH2veMrZVq(ib@f_NAX*qgwHOP6`>+CK|3A4Lyw%ex<{C+x2{Z z1=%NSL652{5@)cHnAhi#$z}WI)r&HIhVrIl##f8Cb0#F&64vpdKey3rLwY`h%cb{N z2A3`T`kr;sv!gV73Z;66x(UxGk`3S=?*Ia#C#G>9~@`Gx~t`k_)%o z2u*BP;jyEQg-Gw`m8g&;?UOSNjlzB;UY0U%`s3CMw(ah~;481KJ7Y!){Zleo@4rk8 zGVAlqIwVd^Zvnf4yOmkz`r&6D-ItCn6!MQ!4!hGct#|XMoVFD)gL;F?ulz|=Ro<8} z01iyb&r;MH8t^QOdHKn5DCgBO>u#yo!kH2Z?~<3L^*FtzyI6+*2Q6?$y*_rOVZfx& z$tTY5VZm$fD0p??a@dEYt!bf4LFraHf?u6^PwEz4;r=?xbea7dqoK=tSA&XZ9@pP? zZu_R<2+BR3U-I~d9Ck+(na%Z&yz(zlMF!&z#VST|TsFT(xZ$Y1m=Iami=imUTnljmh$Jux=KuE3Q|S^iahv!lp4ML7re^nBGg?Ygtt}_>xu1&c{SKlmER9RiWp>AIH}cl^ z!2ny$DTdhX+kB5=!n|J=uaM62je2-cT`m_Mr-v*Y^1YIsuY9v@|H;KW#FJjGt|Kk6 z$K2?g_vzL0^S$UOwmf7?Jo^~eJFkHqgEXf&+pIr0B>r{IT^Xuhmx| z)<4hQH!dRR>{PnmSCV06+0VJ7fQqQvmW%dva@;ddPZA%>PaJ9W?+>xHIiX;u^A#O* zFGTgMew=*W)e*|hx(U+-<34$5ZINjycXa~ftFsm6cW!dgyEXc&UpfK@odg?7Sk%+T z-ftlf>=YD%@ppqQVaD2fl)xX7uOscXd>8}Dnd>xPT^Kn3rBk1QKuK=(R(vCQKdf2e z8{W7Q?Z%$|sEnVji1@MN!hs1#`km{g+qD5NCxbh=)CbAnLq3|4aUTm)E&)^jYOU$9ayQzN zA=abLOyWw8H?>#Cv%F&M+erHg`_IpcQS$NZs^z}6Z&q{*)p~W@N_`pV-ryc{oaOdd zulM`aam;Mc`~q!6wV%KwQ4*VX`4`3*T=IJOb+N_}@G6CA-Y6*%N|MUs8;<&maWm z*~KMjnjL2i)HIQyuH>@@Q6?cZ=)TMWU$_p!}PoOC0!ujSk7UA>FT<<`PbPw)-2wGas}iJ(c_2 z?t~7~oafT+qY3e(uhWzCtOhi54<3eE&k1_HyKy1k z<-tv#!iDKTQy*{RQ`%is!<=&}s9qb^8v=01@s&qUZ!Ap7l*`57MrKB6?Sk+g|ECvs zE4G;WUg0jfr#)#j^9){0Vg01SSN5=eJkl_(aMF7^(M-WhRX%^WvaZ7 ziKwxX$;(SIidth~?G`t;QEuzfS9`e5H*!>kw(KGY;VN(JYjlS8+baaUHF{Yrbf=EX zaS>9#Yf~kg*;eH~V9oTu$YlnHwG}1`rYJo8c@l7KsxROFbanswetsl0ZmLaxx@%jY znXz!@{o~+06W(OT3EEljJ2T;3zZPqkkOK{(7DGLr%DdzickTqS#HY3m1HGQGzd$ri z9orgiRoHvo{kp)YF;44DgUD)?4e79{(9Bwz@ zk4$|bVwxQC%rU{G33m41tMEjXV$B?_p7|%R{%@}0_mKOb2dI$cFVId9kj|f8d78KM z0QvG)tg~p8j^_@Nz#EGiR{dXIrj%38a&x4@9GUF-CtZJr4blCEGlx>crIhfXDeH(v0N@aIa`YS&C^%ZQxZ~O z8?9L>3HQ#pgv%D#H()HCMSAAg-y zIu3d5EM>TPtTSInpCCnqOYy zdQ@L?2zUAs2%ppDn+j<1ej+LG^_)tT+!ln7!>v(n>^!>O$um+@7A_+|$s<9L!A3@h zZC(PuTqK3uO*yxU8qFzx0ZZ*nMW6WD0~cPujR&R=pScLr-18H~bCLIgYq>^kt0Z#Y z3THQk$=_YAVGs&3`86&Hj2h88OBT-6#}*k&tZ73ZuYbD{3boXH;D3*Iq3Vlt#lqA> znJB16{I5wtDGQIsWu8sR&$@&Y8Hkga98JQWm}7jW1mXX1I0%pH%$7b^D7=O%uVJV! zpQ;Jae1Bt9F(YUC?A!4f)U)7Kq^FMUPhXC%>WV9|(uJCOS&Nq%u3vsxl6Xh^$tvRx z)k>O64Q-(7L^60b#WTD~y4iVqB1A3S&4;l5UbaYG7@k|!0ct03AALKJm|UOgc2t4&Go$%K4}1KT{5!CL%-$W>1Y{TgQ55IS{_Bm|8nu* zH^ij99n&$OQBBReCo>PGc~vLXC~92TAY}eaXJrl7Z267+q8PFqH3CPq+dQd#OAj18j({u`&`E`dwcO1QwPY4iK< z@kqv8n=@ZgC+_At`n`%~JjZ+7#jF8x=|<)AYaM|5Hm=mFFAzGBM-z-$6AT=59(le4 z@uO5GVu6C9DnWVtx-jxx`er-+#e1Jv8*}%{E;QZOQfBq-vi&O>HR0++8TXjGl)7~^ zApfKJ^r!^km}H~fM|i+*+rZSNDAy|*Jwkpg_Bg*}HfUV$1ChLqOU}N->Z&d>Q~t}g z9QOot@5au45unh|GG;3BUp_8$n|iNp5>bi}TzPo%Jj{Wx8e2qPL_pKtdRbX88-%_u zh~{2p4%(|UQTLOnAC)1PM$aUVrhc{1upx$K;Oxu+c_jO48H3Ur;}!HV5!PnkUZ-%g zPR_;RCzHq$7Z<_KiY&i_k zUa?xk=&_66^J)4Srta5H|7<~<`m{UA%3WE{QY(bs5Zw?}-6{|fx|P~%p6dWYHhh&UR1%d%9j0$o+Xndq0nXjINDpg#?qWL zIJ}?GWnldm*xf9i4qnZhkH6XzeR<)~B(^lutuD;Uu${KT`-CrGO!v>%UPsrh_H9#6tTY;*LV(S`5;`e%f`J_76>D~mSQ<<*{r03+-=?7}*-T5H> zTfW3Sf|$;UW9ChNB0g{@K`+~D#2j_)&MS-G35x06+oNhO_YPLS!|}JiTfx1ucPsa% zl1tx6GK&2{2x&xCJ#bg<9x}Xr*F|``>D|E_S#twU2Cj9qzrs94X{cUNa=opl=>cQ^ z-dQ5g;f!;}tKWfw`F3uFJ_eN|bC>5CLX+F>#29-0a!{))3X zQA|M+GpBjgDsZ6t1YF${<88pO9aRzxMZx}~tk83g0+$&S3RB5&?99`I{tip|vzlh_ zck@*Y)|S~wFRw(8oDkayOJ8ZT?Z=L$F9g&m!bUIXy{8=uf1jIpZaZeF13^_s&%}0_Ra|6cKfE#t&$Z@+4dWp=Z3|jhqkfl z4lAb`yTz*QKRKVS|7xG(+4raEF-KWjS%XgDpOq`vc$=rq3Fb58?kg^)f9|{_$v&*j z*3UMn?a|;FdmsO;5*t$Sx?1{4itm}OD1qMzSjh*U)fzo3?<*JU9VPW*tJsbwF8HU!5s*s53JX&g{T;IulrDvxu?kx~ z+2PuBHpt@&1JbS5hfElnS-785ApvR@ih*vitv`QIZGVr(sKG%mjLU*^UJ}k*lbko| zHs+DtYb?}i+cTx%H&P*2%2R7$7nFQKTZ{)3=URb?&tyK~e%e(jrKAvM9h<)Z)}#IB z1nYv<^_N9=q^;t#50Xe$ZY{SCC2p5AEqb&pQ8sQs%}p4+^WV@=irfY8E zrQXUL{U@Tn*%_ecKIB)g*(38Oo=7%6Iy6{OJH02j{uhY3ljdaFiymBU&U@E1ln_4t z!^qS=am#ILSVljf(nur9&$Z}wf#Y@IDFb7-LVJiQUMFmSU;)kvlMTggYCvRmpt$k{v7 zY5j7K!`L4jTz3ce|CHD+C8fvb<-d?5QESM$yPcpcvEPEBXQm@u(*syY*Fu=J>IgYBzH%$=< z`4!ASBvp!L7+0?5`ws%jz2seV=#0wKkuG##mjVKI9cD5jAA!k z;P1;7Y#B~_wFCp^{-%*&*t(FF!Ku2wH4KcUJ@m%n%pgVDWSh-AcMkx7`E}85k&nOL zpOTq(2~TbGD?1Q_?+P-d5JqMck~NAh$%|b|>=w`2h1c>c&nwt1@G+(@MNs18QoUOD z4A$A58(6~!UX|MqmCpl(loK9P}^5@(@B zQTY<~R)6+p`isNLy0$jy&yOVT+!R#^aY;364Lb~Zq@h$bqkiKrusQw5(z^K>ba8r8 z&|YDEPk8gsl-(LTEt2i#6Z755yu)7{sYZQO@4IyC&$NtOWBc$l0~-bLow`<>*Js~g zp-P-(o@$-%FsE5oK_L6pKfJYAcR1H#5Z&x)Pm?I?7R7<5*n1H)|)1gsX+zRa<$w$j?aDYC>GGl?Kg4E&)L_;Deky7FXCh!E{|)Ct*dmL; zi3P23ck6iLKVSc{a8U6NU7UX4!02GC+*1WtYsgaTbP<$mT4+g}d_($hi~mK$=wCp& zmWzD2c6ffL7M3E)`F=wGkNKp@qZ_AhZyD`ngnw4#{c(7y4^l@wIiGfW<1cV;kIK~h zHJNC22C?U})t;*G=fv+z56R{QS5CI%3?kE{i$3Gvuw=lO#1s-p%53^GXL8=v0QWxI zUzuA&ZhtmFH0S04hJp5kWSNSj`2KRaq$L}x-12MZcLK(@A@G$zgiI5K4R5O3f3VBM^dEuG8g^L1pt`Q!Qy zoP9Jfe`z4<6nWD=b7__QSw8lJz{}d{ovqu{oUoV*$cr+4dH>GN!#KQ9rP~*V@^$Mi zmf1&FuO04n0LY4yY(KN@K3W**0)aNH*WPQBnRw537$gS89@ixfY?}5GZOnLWPB3yd zWI(=-sw$2m(P+r%%wC#GUe%)??zo~e;Wr4Lm7_RoNSMnZ5$&G5tjy9iC zSbS*VRa=JE%gCQtwSKt<<)w_lPjJ>{=gATop!{1+3v2Eg;)Gg0vVrlUm`=?#Ul9T< ztfrBikqI$S2zbcswC^CfR?Ot_7npV~Ya6>!^3nUire)LJVwLWAwf?qMqB`{{`+>(y zrd#`u)Hn_I&!20)p!i29(~-l4PhMgfne3`T!y>!%ECBG$s9#($tjA#O? zqgj8bvUW7f-zAcLy>Gi@rlKjgY7Ihv_X?5KO7y1>Hj!x~GYiSv+b7Q@{mL*!aWy}} z-3U2taWmSPr%1uVJfwvr;!llU+)*_7Oo&BW=?S@ z5`d!|Cor|EpA}4`QI?HgCa zfX|Ypw1kz>eYw;>!_4~sc>a2{riWoTf9(90jfHh@Vb<;KgGc{RCVc`V81^v?VQ)+Tn2euUEPQ9v zmSrY>%Ri5+MbXSux-AW>L9~CgHR)q1Pn*lt7U68E!WLP{d~??@3r~~_=)3)#e$Wb) zPmR>C2d$Fa*;G89I37*%v6e!8==-$%L+nMG8drEjFtgEnLuyquevju?AQT zsd(`Z@AJj*GNL#Q3FaEWhQoNZc76q~%q~RxfsYnLKqpYR$8()J?`)2dl%gu~cKA*T zF5=<6Q|S#e&lxhAYg9!f40DgsWehlBfSZT^Sg4Eb=bGOZcw`BOfHd7T%yU4{>Nc|+ zUMp{Xnps<#@N#>}TfHw<_~p)%!j|>X@Ryb5+q6^t9>YLV$8Wy4@4v27E zh@jmsQ{%NzHJ8dJF$VJec_$an^2;f%AOYNRsHNU@?z2B3wAE{Fe(w zYJ;O&{PYEXnxF$ljFW>wL|8g#-9|2!Jv5%+x}VhHkc=hLig_BPT|mc%kT)3DmPbI$ zB`6HNP@-8}CPv^J7!1^!`i5pnF6Xgf87K#%*>Gxp5(A2C+ABf!d1^xQ1~!6#pV=cJ zcJQ$NaYv1W3-VkHU&WnJ$MoNOv6cp3dYQ55Sn~Di^M+tX{J!hbUqGag%ddhOvxQ#M z7Pkg2!#lAs0Q;or!0d&NB5&B2FYCL=pXJwsZ?f0VGKbGUDm&9MeD?mu!)4XVQbtZZ z*T=trc=VXr4Q=SdgfK@n<%Fs^|NK$iXDzcYG-43robh&S0T&+(@S-)o62^Qx+FDP& zJZbK5d+oD=rO)m0;$ZUcptj4^$xNSYk3ATV>9R}qf(53&5gnA|ly7#Q3sNg3(J5WB zU7afrs#k~mEUX5r3o3Q?n1bBPw9WS)h!s6@nkS2LDJC@7k$9WsLyHxCK<0z;?+*ZFy5=zxQ*(mOTl8=~k zvlq3RUzC}YFGuVi<^aQ4BEBNTt_u1#2#r*kSG*0|ui%3VUiz9&v2}Ek5OV8maKUbg z|F+~=8lpB3*`s>o&P@#hb8zQ~mG=aKIMj}rLwN~Ipw0t)L(ve?XQKiA}^0GApo}m z&rP4G5^-8I#Ogo%)(~S4nt+-8_6iLT&TdJ}y|9jp!3d)G9m;({~_Z=y$ zo?o_g8P<$`{RM^+p7IVc_^?9hsnc_2XuEj^Le-t^(;s>R6>BP+{~I$mtDL8*}nKnM2?~ zgr5j{#HdoiN-~g|(2!3CbGxdc1GYY1KXYnC{NtF;vm6C&l zPDp*sVm^yJilT10D09KHzQq1Te+U9Vr-uc7dm1H3R`P7TKTJen>?{H2re&gW;OuR9 zg7>@93${86TD0J8EEdynmoM!(#eu2uE-%2ImW|~f&twXs$PGKxPM1SsU2~It{4Q6y zoC0?z-@x$3LtqjW1a67HG~V)AiM~kL#VLiKLF` zTHwGJ8`|dAF$x#K%|fWEXj;9PqQ-B8dnmm}zDNvcbSG$_8Pk;_ z`JxcpsAF(cdM*wz0FwV6_fD0Gez*;j()npG7*ta2o70mi=-t8IGrHf<{s!aa=dO|c zVB!Yg^8tC$fnRX?oCtTgJIb&ppFZy-f#M7T^9!t7z_j6evCw=DsO(q}eiG_v!(sQ) z+h*#tgCf6VObsakn_nH=^Kpju_DsBU0l!>p@{>a6r6g-1#xeizv;4zi28t#>#-HD) z2%ON!f996?Ep>YMp#f7kx~i~akja$L9&-gAz^WQlEND?U{EE*+s3TBB2zZi2z=_GT z7orLFf?$uT%9;q;VOh6GZT!_zEx-iYjN;35l;t5mCW@rhw#bF;V9R{ zkKeC5(q|eehAwTjI(@c&@=?lC=o0jO90FPfr4*YZm@<12wW>obd-Z0>X@SKtTPHqL+l+WM{8NV+49e><8Ad z+)80aBamMcSCxYHLfc{b;?~nsAJYW5D-BfTS>~qt9B#E#Y2iGYU>LI}pU=eF_njcL zb2G-y(B{XajWoQoebA39ykc1kPImZz*to00P1j1X%HNyW+A>ynfqfhrC3?>%8{>XI zG~{E3&M%i&y8POsYJxJ>NvSimG!HV^J`=X^F#*?48I52w=SOLuM0S=7Q=VO``qX{@ z#sJD_!6d(yP=qp2#ESBMvky0K?&H+*^z2WO#QQDKQQ61 zSE|V^mRn1*67{>#>F6&qK)%yS3yakNDIDNeyr5R1)$%O`ZQvZiMeZf>*GNqj~6_97z(Xw^P19Sf~`HOHz1)`^; zIdM*j>o0K5sc<-4-hWu!crKsNI_xy3WjdC2hyG!$l zzpG@Qe2)_t9tC1#B}qJQPnCHjvodneQ9pj)s|tmX@*FGjHe)+!mpyg4938#pVPKv? zgi9G#>4XIT;GYRse7Tort+LZWNlmPI+_Gbk_?|g7j4x)b&hzC1bkj8@9}&`YAFf&& zz82O0(+K-^4#5CvP`urF7}@6Rz-$d zkU%nwLWY&#`Lq7#mjED+u0fB^|5wF$XZP|>@!_W!d4t4cmwHF0>J(&6*%wkdJRdp5 zmu7W#ww*v zR_)%)263xf;~I>t^9RtsKvYU>ipPbE_FLmjh)oYud$0j}Yq_oLAn@waCF)0^W``6{u%81_R`WnY)JeD0w)9B=iThaRO07p9LDxm*jBmXh%$lXw z33zrl$f!kr*4UtMYSaR;~#i3V|uZu6Yd_tOzi*oD~aLBLFamOBZBAHVKG4ZP);8 z*o-q&Y7zwrRW#`eMqUKeI!IgaWGsra^4t5lfsec^az5kCj3l7mCd-VVm1V(7P`CPm z;;MAmXu5U+%0hi7xu*4Kl>oFm4kNFcO%8SOSXdcUK&?ko&ei9kihNY;7<3o>@yK-q zTT^O+UWW7}~{#E(P32D+&@UB$t>1H^NAIv30!PK!k%2`@>6$)wB2x|$1?5oxjRbWAJR zJOJy?p(Zl!tw*|y^GC~uPP|%moVa+>qu@RzZJ(|~fi=Epxv(FIs|~Ch-mIK|{EgwK z3p@B?O-0-Ez~OrJ2-vJ^+wDAr8KwF9=VqRJ;M1eQS;(bQMX^}^+n!4~bwll3F zKq09$;EE8pGlnT}PC6I%O4D2N`!vPW=O?r@6VGX@#y#Sa4h&WgA3Q##9qg=yH96OS z(%Q~mQ+(7$M-OoPqE)`~6Ge7*SJn&RnRK^ny0Wa@E1HD+J?RbAdJk1d6Dc)SuY;;b`M=oywkqd2+@y;CB@Pp2!Is#{mL<=*@B#qrJuXZA+3kgVFZcb+$eP_puH$b(v( za=F_+5$_i@S0Kpn{^{@YHA{1Kj1^I@#9oq)tGmVo_FTEU2NnOCo%_4Ph;;+m&9Rce zD=92=O1UFlHL!&#H#$;TSx#RE<{?VDZ#896!1nS<1L0MRuaAfWvt$SD8g;k>(nR|Kd zoq}oc?Cd@q8Jjko-RJcpE)^D$x#%wEqV8yP69;tEKO0f+l*up|#v^nao7ir>*-(2+ zIK%Y4@v`1k33#e6my6LS8;%QnHp7=06)Z$kME3gs0=49C{_J_res{UWzN(-=aYDMr zl>MQeflBtGgxZPW$Qht5m5HPqg%CubD(BeLJ?zV-I{b3{KpX`;f$Uf+BreO&l zvCf})9(RJr)3Vipmnn4Ya}*6pTWo$Ek~*n$X}i{K;dqM!6CLZ4YFC>#AZTa2DnTEA z=P|bIRUlelDE|r$(S!qzC^j47H1~aNu?FC~g^uRnC87Kv#;8e~7^@I@&sry`jXv{d z_oJX+wO5;P9bK83dQoY2`rq?H|}lUiSU_OOg9)R9y=Ko5cuN9@eQTUCNG4(iqE zPPFQCCcp2`rH08<Y{~Kp1+ki8kSHS{=W4u&}?N1db@0mI=o{V{~(q*eO$`+ zs=1}kXRcz0xZ1T;cAE1Te!+a5jTY8&-AhR%eY$y*6J})o z>?RE&9l9e5GqyCpn3bt+bo}fWX+2|H5bqnDTEfc1ahEJiM@)siCR2HsvE8BJ!{=0>F6*oqIC?-A>Ckr${}x;aBYB)$0Y(my^Xztu0cysP7Ll>EJ#i$0Zv zZx4$S@Ae`d%snLF`0YbhgY>KJL?)TKHZwyq6P*U6GjXZuiZ&gO%${FA>H2-Jhxiw` z_NQ#O=F1(+a~2;)QoR>dqri7n4LJ0Y>1mVBR?-&-I=L-P6bt@&c(9-0BPomiAa*l4 z15`MMPlSPKC?j>b^MbEAIdwFn-oUTdOpmLow=7>@3yQsBF|&205gz9x*5_)Z@F`Qn zV}qcUz#tZvDe{3pk7=8}C-~+s@YL}WOzO;8!M+)0u^yp7L0a|37b;TH%UyqdH`u%E zMPuAod-Y)fe>W?Wn|?tMv3pg9)G6i!@9*Cxo))e=`7IPGJKksLjD%Q8spEtS-Bx{D zK8>%`&MN!`Ub=@1voi;s5Nm})us_02-u;|-!-{Z-`U|85% zX*gS3+un(JESi{VrjD2@#7u=!Gck|tAxI@fjn(651*N3atVIz+3^g?lik|k+iWnL- zrskSk9XN(M)jIlaec$W*@m<&2YwvaKpU+y)TI;^==gx=Js@`&-*6^;bB_FzW)3@3) zjaIXBZlw~>?%-INGb1)S8WHoZWh$)J!Eayg)$6^c09W-FsyhsOe*mo$GU3tDVQ;Ve z0A}#Kul`QXlkCy^P@7dz^VRbBvV-BdgeD|heDv+d({oUP!+X71#>xxnb1T0kuBs_2 zK@WjiJA}~8 z55vzdTK_I%Hi!ArsM5d$((W05h{2Ht#;2r)76o9*GR^Tm^LD9N{ueg_7E9A(1De&3 zm;)bOEU*Jb{#kK>)0MH(&ko(v3tYB91oMtDqHk?&LIBbG(Rz}95^`%ON@B>d#XE`r zqA>!6dTl!KPAyFTvnxv9)<@lu(EFa@Ue$i~H?6%DN|#`6tcG+Zig!)DDlk7OYTGAj zW>(M7@t5Zx#;h1srHo6Rt*NTZb31aSgH&5>F+V;H1%_oe@qeAg^OLXVi9)y&lLA~7 zNJ(fpg^@Y(oVLiENLvzzKhl8)@$am~CQgSM!q{%pz%CVpm7p&p9Dn|Rpgqwlw)W%I z&m#(_%btOX8xP4;d`EhOW~xzabI02Y?H#u{hp%n*?O~?JFBazE`5L*M%qBjo*`&;` z{YyYkcZ4n=Mgfv?j%uOHZKJ8HT~?wn9WK$hzJ9$2h_jg&8q6skJ|BJ}pGq zbtWuI+cDn{W?NhTQ&yLE1x*zZvGF0VG0Rg#-d*{8j{qhh)LE)DA+dA!pQvq*E4@BZ zlZMd@-0+2&8Lr(bJ&B-;;lWRx`0`-9j*GoZW##1rA)bfL4ZkVe`))o!S0?Mp983Q7 zubZVC;EN2Bo$5yU*v2K9{s8ovoHaujSg}MF$Q`{e+sCNP*Cv@x@&%d(F0pNa65+p* z1LwFinz$mX{fWn}ysRyPBb*KH8~UE-B*qbPTjepQYorQq%B1Xt_V?2lzSQB;iwp0w zGh=e}Ig$)D_8XtZ7TofJ417?wZ4I7hE}pKXOyy?@GiN`Zf%o4;Z!#<>M*O>`;a%4E z{S@rAJyBS7fc!?brq_x$zvw zyIvK&=-tQi6UDQm``12=j=x&ok_fm*GG>D!*JF}tE%J}ePw&!+n$H-QCyh+h*7qmU zS6vH|LzS&1ute>IwvanMohI8<&wv;AI8ClPWm&!cI?B3;@Z4&9(Cinp0=pa5@yjJ0$Zw&CRJ7m? zOXS;;3}m-}W#?=7GvN|mkKvdzngwYVkE46YyjGU?S9SFz1`^zw{-Dg2qA#x>Et!6w zkEYuI>+LseKDPE$r3bl7XMu;3?yk0BuWjKE=88f6{3{%idSqV5(?{Kzf1KJnHP#Tf z(a-!`&@Ai4odE^e1rs6fk|vFA^^KH2dw>sy%vw_rLLOA?05gb5p!u1kkW=N$&FO8! z!uMb4P{T%|I*5p`gclJhkh zB+F~%`c0DF8#7@(1N|HTcd)(KwwMEgvi5Mlr^c#>ebt4|<(tQlkp@^4A9A`$wLwPT zkFfnkac{e(KF|LpzhI{Ny84Lom6u3bz{p&+es81S^d4fs4c>!tZVjzE{(ma4V@|`no};fprktC#y-o$R~F9@58UYhGpVxrCoMdu-C|S zw#a$umj!sGRuG#W!E@5+QRAFzna`-3K=iRxK0Qc9tI;Vr3;FHJXk0nP@AdVd{E#7< z=$v>$bVB+Ef_Ino^j7AYiv?H4X;C8)UzX|pW*pu5-&yl_DVtw^01uDh{AX)^ea^gp zRc2Kl&w527You%V^VocAS*ddF>42~rAqU28^24Jl7HywrUaZ8VNE7Z1331j-n9kWa z{sJh-wTSu6r0ZWceuj`OstO@SSW*02z2eQzFzD-fQ$+o)BUbiaE&6&AMKsg z4@DZ+S<+u|ED)$rdrsuj!!(YeciT5lwJ;y`+vfprvy3HP-iPBD8O8|}0Uot~c0AVT z|KEzuKqsFi7l|D-Ts^Qy8$PNa4!oH3QJ8$9A&lbf3)yJ4WuNG&W@A@4YIyTzyh;11 z@*0WV0^*sbr!bYKCUs%_E}x;m<;%V(L)YxuW`n8Wk2-f8bDfU#Dr6$HLK@@Wy=qE) ztL-{(`|qlwlVaxg2cMVQXQbxewZy$UdSkrlDRGw}(+_!+lN!K43NNv(OVaFvANV1V z)xiSpMkK{hYD}GKMWkEzq?V2YpYz#l%{XyQ(?<&azAl3ds603udvLS2lprPeuVl`H zi5;}y=Xy>uC)H7oC{HsY0a+k@BJj$Eiaw1i>aL+%3?fSE9&cIO@u7Wh>Z15p>$R-d zK#&C1)`N$&0n`pUAE|7FtuScs)L_}oO&wuW+wj$f1CL1Oh2OtTeJsTp-e46bi8~~- zq}0!A8}N9LnA*5@w zB~d?MtZV>KRm-q-VdR)PuAwYYjw0DnEKZaXh^^*B?@Ixapl?8_=KG)m&sec$(rZ3E z*B6UpS0SuLg@7p|Islrzb;V~BS-UuRbk#~+RYX{n37ajVcQM4s9JoAGpE@4&wy+1G zqLO03+iny(jw4F`Zy2MC0>HFjP;&D3VT>M`twu~BCPq4q%psoM>?euKNaz9eY=?E; z2ct1vw@(I|R(p0gZF=Jt%}kcCi3RIVUK2c~+IpG0=Xyrju2I+4NIC5cM8f)}hz>gMZox*(gD@tW7C2##u-Nc=IyMWwB$( z&gX$%AUs(N8C0qvs)!sbSsV(9G*w~iS*aoi=5$%Z6W1grnbZou*8`IE8V2^7USWdP zL;Z1ilg!@<6qse$pZg5-r|Saqmf|TKN+JxS zBqldux{+AgRQSvvp24H~H3lsj`ip3@RxTbgCw+SJC2{=8m`a9Q4_Wm(2TkW@<%44& z!FvtQPxp*>yEZ%P@|CoH#|BDwh<7i48T|Hc*5A#3;!yfYOq`6>I97iF`R6Vw^KQz$ zw68TT<4hA;rk}H9PG`Jhbet*Y5d-TK-m2r6ww+}G20!83muwE0uWIIn6+ZBuAbzTn z`xTVfdag9q$z*pYtCIAZ2Eoo<^-ee0Iau9Hbftn~E(6NjPN-i zhLU+zO0{LhXknLq(qS7!y{j(-)8LePFS#&Td<}=LJQN=WV_&JuRqJ!H2lP7d`F(0< zfΞ{61I_n8m;>hk^X`7lP%3`y-dNsoB`cE_o76FgqxTw&eqD-0VgOc^OygW16+O zU%~lQin8da<_+~)QSNuJ@4<<3lq?tQ3y(-$iOJb`f2;b_20ZWqxOVmu2FKt`Wi!@; zn`h?LDCH1g4`8)f%2DIFE0=l_ot^a;>_xKKOq>9yqql{`r}q$)ukI^Cc9CZtrB~ZL zryOL26fqE_AdH5u)#58{2NgU62>taqs&L7-fZ?D{M|X_68br+@`C@+|S+lF756HL+|H# zutr0iunngA9g4azxm>tln=<<3~4ylyob!4iX^qVtdqGj zg)buAI|4CU`b~$M9wZFgsXz9{TMna}*|eXCZF(?G21GwBo@Dv03?|{@7_=W+_ClnYABMUA?5L2{9(|+;_ z-2V+B25CxhXiK#hxNgK<#^E$M9wfI=tr-~Mt`;UBh;d_Np65`&HRAUga-I~McZ7Lc zwp-YSaXHyB3OS(nC0qnq=_K2fLxSg6d6WjmY?pxW3c!MzoJTnq&yy8o3Rfzp)bFk- zz4!qb0R8##{lzoCDAt)l7`laeus)-pdu+BstYRpLoFWJ63tJ|Dvdk-{X7FNIp+-^8 zPv&WW*fD7b@y2(_RNL}f-_0A}uGmT>?M>G;tK`zRT`LX=<= zBVVZR0)a=+rcDh!O(5y?*2ue^QE@Umg!MU1VPh|}P0ba|-yvNv`i@B)OIPe@w++a> zzEO`vMTVTB_;y|D64mxlq;b11?bpbH+NQ3bJfzmoBk0edoh-nUxcv+w=Z~5H0Qn&t A1ONa4 literal 0 HcmV?d00001 diff --git a/models/Reference/segmind--SegMoE-4x2-v0.jpg b/models/Reference/segmind--SegMoE-4x2-v0.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7d7989ec2a1c0825b092263fe09324ed9a995119 GIT binary patch literal 21996 zcmbTdc|4Tg|37@qFvw0r_MOI>Wyo&qL)Njx&|+tjvQ|nBvNYDQ#Uu^JFt#YW6xr7- zLy}6eE4>RzO25nd^Z9=7$K&_cecu=7T<2VKjO+Ehp68s`cKx0Gy8v)oTUc2D5C{ag z2>t+nKLWny*Ms~4z}6N}004jmV1x((Ft7yyiy=h#f7=!i1ifbcjsZUh7V|{%6`~w1maN!Y=QP-~D zh>pLTkeHNwkC2s}lbd%xzkpOyN-3k3SI{b-JZ)%fYHn$5d)eL7+t)uZI5as$pPqR= z`)2Or;-{tMmDRQNjc?z#cXog5{oMa`Bo_pf^M8*22=;%;#RJL(Wn^SvWImD$0=)(n z1|CM3q#6^iu_Lor=y56aSQfYmvADkLn6w7wA3pD}308g?JtKXT0h>Q;^of>?6Ivc3MKUKJbDm^nvZfMkf&W${4#{kJDvf+B zXUh1fa-I~_C3%y8hqh|AfSZ#ejm*YM??vJtVoRM+DUeE4I=)V*gab7Usg$6z)CrYx zppGY2bUR30o#w_}wEMTuR1xpM__or3&T?Oqe?x__wW8Vas_Z+*kdg&7wnQ1#pI4FM zz>wnXM*|k8d2mefxYR~KQ7E-hGG5bS6vZHvM#3{`3Tvvx3wa@d)S%Qrh`^M# z+{Q$WnvLS{LEEYd^^y8w!4@6pqaZtj$m0wgOtI`pApmSb%imM%J5c=h%h)C<#dHU? z8e=lcTq0HKL>bOE(?D^WC4q*`Pac)0N@*CAnI=%I2zVm8L0K}ri~_}s1M)Cc8F(CC zhAGp55kJXlD(i&i01+c`05((-6))>#>xB00JVMO*Kd`~a)g!^1fczIb3l0e-2FFD5 zv2JIt3h32o0UYW-&?JJ;!0;ecK%G!A#}S|sbci%_NFZc)r>v;Z z{7DW+rhjpyK~!rL~voSlU6s6-IMVhwZ z<=#1Lk&iVy>s3H8)5mZ?W}^uYP3Sc?X8t+jjF+wHEUDu`_>x$(&rB}PLK{P1 zl2w+Fg0q%bvKpfXVG`tH|0&2sG=~Hd@*fbVj46!dBW@W}<3TvViZ|YdDK;zKmI*J* zMBGS;|a&w;U>=AFmx9hmMEarPZ0rJWa++ zJi-D|8!yDt^hWD%Z)vo{v5RepkeaEFi{pr~*P=1&-|NFm+5OtsUpp}+u)FE#^dt8L zS|^5hEwP&?%$hluQnz+XiAlOTN~HC~SRq9qJiAxLRb*}D^jW*-Vt0p9-!VIG+UJ4; zCa%akytO-4JEJGy<4|k?GqgArm~OuF<_yAIf46q9#9XkmG*f5n^nuEid_Emf#Ttd! z`dI6(k*gVuRVVWtw#F2j4)m4YHl-4j41EY~4O#E_9&|*h&vt1rNX0SROk8a3_k=11 zcZvL7{R@cIb~L=bA!ud%J>~-r7M|A76tp^P46u_vPuV7_j}T4^DT(y+&U{@NfZwV> zk4Ao9TbJO3ktipYKJt9-5>utLsd= zMzNCX4FJA1eynYJ{)={-R+08FTfJ*1AJo6J^b1$aNk1l-5MN;#qn?J(C$ycgLR6x` z-CmOh;wQbA<9Y>snUfF>c&Diq3MLCE?2;oX>YvXclFH7=A9s*vH?~hAwbz%ycp)`^NnbAFm%Ri>RY5tcyxO86goe|7f7tNX?uDH+8qt-wo{c zA=q-m&;AAKHQDdg6 zTdAHavPE&Ofy*M0*ZQx`bVOVbQa9dU6s+5L0?Bw4_o~r{mjm~uH2Z{1kdhe6(pfF+dbS~4ElAyzf8sJJ4rlHU_*DtX7bC%b^WEN zvXTCw;Oe1E$7+ZAOdfVljEIz@SPV85`gVrz8d%n7N`{f%R8F2#h#fWba=Vqq@zFLw zb1}J~?WqXsgY@p&Et|kCrmI(h4O~tbQuSG?DSE^@{*UM7%WlEJjSvL{1Y+@+kBxyBt)17MJ6hBjU z?X=Yi_JkHbc3V=`XFE>4$;w#zxRhHVZJ^9~6w|8Gn_BZyV4j`m=qtHb18*-FY{>9{yv$&zI3o)u8pvY-Ct%OMI6SQL?m*|=v5fibA+C=-! z>awDBou3eW|HDk^(Sj+3bER4_Om<*+*f8LNc|=TGYdS=bJPv?bjXpT?IvE3$;1vkQ zuIP^_3PS=kHUrddKyFI@_YCM`Sb^&xFRvrP>l{c2wS`IW0&)Vy%SsxXyfk>Lg}e38 z8KRkxy0^{p(eqdRnqt$GlGl_{CtH&Aiosxc$kKL{)D%+hCgTUm50V@9J2VxQGsm+r zt9*5InLTBza1kF?n9muVFQfjOhKYpOsvi?y+~k&X=m$Y3KJ!c{ztrwVNY(Kkh}vqp zXRV1cUVY0nPaywtmAa6pXS~wja%DWpytdrZUG+TrnRbnHt@lSC+a;Gx_opN7*O6Ms z`2E_Pg`f7{33bc3I`Bbl|Bk-%e65p-$J%V|V2=2YSA4NITQA6P)VvbqY9U9R_Uqs| z{q@9EqrN1*;kfQ{DSZ92wk+#wHy-nAOQf#Uk7xIV5>Y2=Wdkx*vrcyn_zC>p$YOug zv5$dnj!jlD!VF70%=U$9;I(`o4^JqYMXdHsbn%4IL+xpQBYoK!mI31?5X(xl8Y|+b4-Kp^PS`lfE9<%#AsrE#Y zC~R0liJH@(y9Q5dIcKS*rgLa5IymQQYi4VnMVwzaRa|+tS&&h!7#DPj=SB}t`_?Zy zdgz7|aqUv%@nbeq$-vM^Sb57AN9+Lf%lhX_X2g_i%}LCO+KCO0@ z*T0;LGxole_-?Y~z&9-8eNXux%D(+i2MGLAP~2@F1Fi_elKerDq&@YJzkpw&@L%Au z4a;);U}jwCJkNNBqm#w8Q^ZrlfBKAG-gz0zU5f7}_Sa~cUtdzNm=QGx+&CJxn>Q4= zG|CM+0tGMoN$iMwy7&&CX2PG6Nu5t*o^=!w(?~7a2Q}-uF6s}7KL}_ zMs&1U4LgF8g9NpAHZhmy4D7c{y2A7?`jVo$r`!Gl3Q8xqeGf2`0uXamo<8$aSd?q7 zdby^KBYj*ie@`trv+!15&3u2mk|i|Tyw5^mVCaJmb)I20M4xrAXCt*1d%TIc48=8^ zJz1VZ9ZeH6S+V#Gv!tj$(0LTagj=i*ceysc-1sAT1H;l5zbzj9xKjUAR3B%-(ywW0 z^)gwt6ssc>58oVeozlFH88WX&Q5-g`K~r&xyRVp*e6Ipmf+m6Xp@uQ_NC^d!CrN-Y zilVj%s+Yac%+5&?$`Q^w9=pcI4fCwSw}(g-5B181jg~b5i;b5T)_Z-I0!wt~*x45w zE+(bto)%4Z75PyEkDp74MR^>!=Eu)v3N9)|)iA)+1YYy+zv3%b(vK($E$n#rm@(D^ z#70aZC&4FAg4R`af2LRXDqm&pAg)(6NIp4I7Ai)~6au#N<^BR|*SGgFwjO<~zQT5{ z7xu=AA>{Vmh~hxG^tjzgJu2g3@oRiqR`Av9nmIGn9TKi{JH(Tz`kvhWHTh}}UHCDT z;XZX?O^cxOW=5Kd{JE(I?eWr|B!p;2yoHC6&!kc}tV&H_Dg~mJp)QvT#^W-G*uomp zgiUCZ^U@veikVMWyEy|To=h!7E>@g539g!QSq}X*4(j1uNS2nzhvStK{66C`fza=& zuNZ}bgHf9{Y~PsgFH2veMrZVq(ib@f_NAX*qgwHOP6`>+CK|3A4Lyw%ex<{C+x2{Z z1=%NSL652{5@)cHnAhi#$z}WI)r&HIhVrIl##f8Cb0#F&64vpdKey3rLwY`h%cb{N z2A3`T`kr;sv!gV73Z;66x(UxGk`3S=?*Ia#C#G>9~@`Gx~t`k_)%o z2u*BP;jyEQg-Gw`m8g&;?UOSNjlzB;UY0U%`s3CMw(ah~;481KJ7Y!){Zleo@4rk8 zGVAlqIwVd^Zvnf4yOmkz`r&6D-ItCn6!MQ!4!hGct#|XMoVFD)gL;F?ulz|=Ro<8} z01iyb&r;MH8t^QOdHKn5DCgBO>u#yo!kH2Z?~<3L^*FtzyI6+*2Q6?$y*_rOVZfx& z$tTY5VZm$fD0p??a@dEYt!bf4LFraHf?u6^PwEz4;r=?xbea7dqoK=tSA&XZ9@pP? zZu_R<2+BR3U-I~d9Ck+(na%Z&yz(zlMF!&z#VST|TsFT(xZ$Y1m=Iami=imUTnljmh$Jux=KuE3Q|S^iahv!lp4ML7re^nBGg?Ygtt}_>xu1&c{SKlmER9RiWp>AIH}cl^ z!2ny$DTdhX+kB5=!n|J=uaM62je2-cT`m_Mr-v*Y^1YIsuY9v@|H;KW#FJjGt|Kk6 z$K2?g_vzL0^S$UOwmf7?Jo^~eJFkHqgEXf&+pIr0B>r{IT^Xuhmx| z)<4hQH!dRR>{PnmSCV06+0VJ7fQqQvmW%dva@;ddPZA%>PaJ9W?+>xHIiX;u^A#O* zFGTgMew=*W)e*|hx(U+-<34$5ZINjycXa~ftFsm6cW!dgyEXc&UpfK@odg?7Sk%+T z-ftlf>=YD%@ppqQVaD2fl)xX7uOscXd>8}Dnd>xPT^Kn3rBk1QKuK=(R(vCQKdf2e z8{W7Q?Z%$|sEnVji1@MN!hs1#`km{g+qD5NCxbh=)CbAnLq3|4aUTm)E&)^jYOU$9ayQzN zA=abLOyWw8H?>#Cv%F&M+erHg`_IpcQS$NZs^z}6Z&q{*)p~W@N_`pV-ryc{oaOdd zulM`aam;Mc`~q!6wV%KwQ4*VX`4`3*T=IJOb+N_}@G6CA-Y6*%N|MUs8;<&maWm z*~KMjnjL2i)HIQyuH>@@Q6?cZ=)TMWU$_p!}PoOC0!ujSk7UA>FT<<`PbPw)-2wGas}iJ(c_2 z?t~7~oafT+qY3e(uhWzCtOhi54<3eE&k1_HyKy1k z<-tv#!iDKTQy*{RQ`%is!<=&}s9qb^8v=01@s&qUZ!Ap7l*`57MrKB6?Sk+g|ECvs zE4G;WUg0jfr#)#j^9){0Vg01SSN5=eJkl_(aMF7^(M-WhRX%^WvaZ7 ziKwxX$;(SIidth~?G`t;QEuzfS9`e5H*!>kw(KGY;VN(JYjlS8+baaUHF{Yrbf=EX zaS>9#Yf~kg*;eH~V9oTu$YlnHwG}1`rYJo8c@l7KsxROFbanswetsl0ZmLaxx@%jY znXz!@{o~+06W(OT3EEljJ2T;3zZPqkkOK{(7DGLr%DdzickTqS#HY3m1HGQGzd$ri z9orgiRoHvo{kp)YF;44DgUD)?4e79{(9Bwz@ zk4$|bVwxQC%rU{G33m41tMEjXV$B?_p7|%R{%@}0_mKOb2dI$cFVId9kj|f8d78KM z0QvG)tg~p8j^_@Nz#EGiR{dXIrj%38a&x4@9GUF-CtZJr4blCEGlx>crIhfXDeH(v0N@aIa`YS&C^%ZQxZ~O z8?9L>3HQ#pgv%D#H()HCMSAAg-y zIu3d5EM>TPtTSInpCCnqOYy zdQ@L?2zUAs2%ppDn+j<1ej+LG^_)tT+!ln7!>v(n>^!>O$um+@7A_+|$s<9L!A3@h zZC(PuTqK3uO*yxU8qFzx0ZZ*nMW6WD0~cPujR&R=pScLr-18H~bCLIgYq>^kt0Z#Y z3THQk$=_YAVGs&3`86&Hj2h88OBT-6#}*k&tZ73ZuYbD{3boXH;D3*Iq3Vlt#lqA> znJB16{I5wtDGQIsWu8sR&$@&Y8Hkga98JQWm}7jW1mXX1I0%pH%$7b^D7=O%uVJV! zpQ;Jae1Bt9F(YUC?A!4f)U)7Kq^FMUPhXC%>WV9|(uJCOS&Nq%u3vsxl6Xh^$tvRx z)k>O64Q-(7L^60b#WTD~y4iVqB1A3S&4;l5UbaYG7@k|!0ct03AALKJm|UOgc2t4&Go$%K4}1KT{5!CL%-$W>1Y{TgQ55IS{_Bm|8nu* zH^ij99n&$OQBBReCo>PGc~vLXC~92TAY}eaXJrl7Z267+q8PFqH3CPq+dQd#OAj18j({u`&`E`dwcO1QwPY4iK< z@kqv8n=@ZgC+_At`n`%~JjZ+7#jF8x=|<)AYaM|5Hm=mFFAzGBM-z-$6AT=59(le4 z@uO5GVu6C9DnWVtx-jxx`er-+#e1Jv8*}%{E;QZOQfBq-vi&O>HR0++8TXjGl)7~^ zApfKJ^r!^km}H~fM|i+*+rZSNDAy|*Jwkpg_Bg*}HfUV$1ChLqOU}N->Z&d>Q~t}g z9QOot@5au45unh|GG;3BUp_8$n|iNp5>bi}TzPo%Jj{Wx8e2qPL_pKtdRbX88-%_u zh~{2p4%(|UQTLOnAC)1PM$aUVrhc{1upx$K;Oxu+c_jO48H3Ur;}!HV5!PnkUZ-%g zPR_;RCzHq$7Z<_KiY&i_k zUa?xk=&_66^J)4Srta5H|7<~<`m{UA%3WE{QY(bs5Zw?}-6{|fx|P~%p6dWYHhh&UR1%d%9j0$o+Xndq0nXjINDpg#?qWL zIJ}?GWnldm*xf9i4qnZhkH6XzeR<)~B(^lutuD;Uu${KT`-CrGO!v>%UPsrh_H9#6tTY;*LV(S`5;`e%f`J_76>D~mSQ<<*{r03+-=?7}*-T5H> zTfW3Sf|$;UW9ChNB0g{@K`+~D#2j_)&MS-G35x06+oNhO_YPLS!|}JiTfx1ucPsa% zl1tx6GK&2{2x&xCJ#bg<9x}Xr*F|``>D|E_S#twU2Cj9qzrs94X{cUNa=opl=>cQ^ z-dQ5g;f!;}tKWfw`F3uFJ_eN|bC>5CLX+F>#29-0a!{))3X zQA|M+GpBjgDsZ6t1YF${<88pO9aRzxMZx}~tk83g0+$&S3RB5&?99`I{tip|vzlh_ zck@*Y)|S~wFRw(8oDkayOJ8ZT?Z=L$F9g&m!bUIXy{8=uf1jIpZaZeF13^_s&%}0_Ra|6cKfE#t&$Z@+4dWp=Z3|jhqkfl z4lAb`yTz*QKRKVS|7xG(+4raEF-KWjS%XgDpOq`vc$=rq3Fb58?kg^)f9|{_$v&*j z*3UMn?a|;FdmsO;5*t$Sx?1{4itm}OD1qMzSjh*U)fzo3?<*JU9VPW*tJsbwF8HU!5s*s53JX&g{T;IulrDvxu?kx~ z+2PuBHpt@&1JbS5hfElnS-785ApvR@ih*vitv`QIZGVr(sKG%mjLU*^UJ}k*lbko| zHs+DtYb?}i+cTx%H&P*2%2R7$7nFQKTZ{)3=URb?&tyK~e%e(jrKAvM9h<)Z)}#IB z1nYv<^_N9=q^;t#50Xe$ZY{SCC2p5AEqb&pQ8sQs%}p4+^WV@=irfY8E zrQXUL{U@Tn*%_ecKIB)g*(38Oo=7%6Iy6{OJH02j{uhY3ljdaFiymBU&U@E1ln_4t z!^qS=am#ILSVljf(nur9&$Z}wf#Y@IDFb7-LVJiQUMFmSU;)kvlMTggYCvRmpt$k{v7 zY5j7K!`L4jTz3ce|CHD+C8fvb<-d?5QESM$yPcpcvEPEBXQm@u(*syY*Fu=J>IgYBzH%$=< z`4!ASBvp!L7+0?5`ws%jz2seV=#0wKkuG##mjVKI9cD5jAA!k z;P1;7Y#B~_wFCp^{-%*&*t(FF!Ku2wH4KcUJ@m%n%pgVDWSh-AcMkx7`E}85k&nOL zpOTq(2~TbGD?1Q_?+P-d5JqMck~NAh$%|b|>=w`2h1c>c&nwt1@G+(@MNs18QoUOD z4A$A58(6~!UX|MqmCpl(loK9P}^5@(@B zQTY<~R)6+p`isNLy0$jy&yOVT+!R#^aY;364Lb~Zq@h$bqkiKrusQw5(z^K>ba8r8 z&|YDEPk8gsl-(LTEt2i#6Z755yu)7{sYZQO@4IyC&$NtOWBc$l0~-bLow`<>*Js~g zp-P-(o@$-%FsE5oK_L6pKfJYAcR1H#5Z&x)Pm?I?7R7<5*n1H)|)1gsX+zRa<$w$j?aDYC>GGl?Kg4E&)L_;Deky7FXCh!E{|)Ct*dmL; zi3P23ck6iLKVSc{a8U6NU7UX4!02GC+*1WtYsgaTbP<$mT4+g}d_($hi~mK$=wCp& zmWzD2c6ffL7M3E)`F=wGkNKp@qZ_AhZyD`ngnw4#{c(7y4^l@wIiGfW<1cV;kIK~h zHJNC22C?U})t;*G=fv+z56R{QS5CI%3?kE{i$3Gvuw=lO#1s-p%53^GXL8=v0QWxI zUzuA&ZhtmFH0S04hJp5kWSNSj`2KRaq$L}x-12MZcLK(@A@G$zgiI5K4R5O3f3VBM^dEuG8g^L1pt`Q!Qy zoP9Jfe`z4<6nWD=b7__QSw8lJz{}d{ovqu{oUoV*$cr+4dH>GN!#KQ9rP~*V@^$Mi zmf1&FuO04n0LY4yY(KN@K3W**0)aNH*WPQBnRw537$gS89@ixfY?}5GZOnLWPB3yd zWI(=-sw$2m(P+r%%wC#GUe%)??zo~e;Wr4Lm7_RoNSMnZ5$&G5tjy9iC zSbS*VRa=JE%gCQtwSKt<<)w_lPjJ>{=gATop!{1+3v2Eg;)Gg0vVrlUm`=?#Ul9T< ztfrBikqI$S2zbcswC^CfR?Ot_7npV~Ya6>!^3nUire)LJVwLWAwf?qMqB`{{`+>(y zrd#`u)Hn_I&!20)p!i29(~-l4PhMgfne3`T!y>!%ECBG$s9#($tjA#O? zqgj8bvUW7f-zAcLy>Gi@rlKjgY7Ihv_X?5KO7y1>Hj!x~GYiSv+b7Q@{mL*!aWy}} z-3U2taWmSPr%1uVJfwvr;!llU+)*_7Oo&BW=?S@ z5`d!|Cor|EpA}4`QI?HgCa zfX|Ypw1kz>eYw;>!_4~sc>a2{riWoTf9(90jfHh@Vb<;KgGc{RCVc`V81^v?VQ)+Tn2euUEPQ9v zmSrY>%Ri5+MbXSux-AW>L9~CgHR)q1Pn*lt7U68E!WLP{d~??@3r~~_=)3)#e$Wb) zPmR>C2d$Fa*;G89I37*%v6e!8==-$%L+nMG8drEjFtgEnLuyquevju?AQT zsd(`Z@AJj*GNL#Q3FaEWhQoNZc76q~%q~RxfsYnLKqpYR$8()J?`)2dl%gu~cKA*T zF5=<6Q|S#e&lxhAYg9!f40DgsWehlBfSZT^Sg4Eb=bGOZcw`BOfHd7T%yU4{>Nc|+ zUMp{Xnps<#@N#>}TfHw<_~p)%!j|>X@Ryb5+q6^t9>YLV$8Wy4@4v27E zh@jmsQ{%NzHJ8dJF$VJec_$an^2;f%AOYNRsHNU@?z2B3wAE{Fe(w zYJ;O&{PYEXnxF$ljFW>wL|8g#-9|2!Jv5%+x}VhHkc=hLig_BPT|mc%kT)3DmPbI$ zB`6HNP@-8}CPv^J7!1^!`i5pnF6Xgf87K#%*>Gxp5(A2C+ABf!d1^xQ1~!6#pV=cJ zcJQ$NaYv1W3-VkHU&WnJ$MoNOv6cp3dYQ55Sn~Di^M+tX{J!hbUqGag%ddhOvxQ#M z7Pkg2!#lAs0Q;or!0d&NB5&B2FYCL=pXJwsZ?f0VGKbGUDm&9MeD?mu!)4XVQbtZZ z*T=trc=VXr4Q=SdgfK@n<%Fs^|NK$iXDzcYG-43robh&S0T&+(@S-)o62^Qx+FDP& zJZbK5d+oD=rO)m0;$ZUcptj4^$xNSYk3ATV>9R}qf(53&5gnA|ly7#Q3sNg3(J5WB zU7afrs#k~mEUX5r3o3Q?n1bBPw9WS)h!s6@nkS2LDJC@7k$9WsLyHxCK<0z;?+*ZFy5=zxQ*(mOTl8=~k zvlq3RUzC}YFGuVi<^aQ4BEBNTt_u1#2#r*kSG*0|ui%3VUiz9&v2}Ek5OV8maKUbg z|F+~=8lpB3*`s>o&P@#hb8zQ~mG=aKIMj}rLwN~Ipw0t)L(ve?XQKiA}^0GApo}m z&rP4G5^-8I#Ogo%)(~S4nt+-8_6iLT&TdJ}y|9jp!3d)G9m;({~_Z=y$ zo?o_g8P<$`{RM^+p7IVc_^?9hsnc_2XuEj^Le-t^(;s>R6>BP+{~I$mtDL8*}nKnM2?~ zgr5j{#HdoiN-~g|(2!3CbGxdc1GYY1KXYnC{NtF;vm6C&l zPDp*sVm^yJilT10D09KHzQq1Te+U9Vr-uc7dm1H3R`P7TKTJen>?{H2re&gW;OuR9 zg7>@93${86TD0J8EEdynmoM!(#eu2uE-%2ImW|~f&twXs$PGKxPM1SsU2~It{4Q6y zoC0?z-@x$3LtqjW1a67HG~V)AiM~kL#VLiKLF` zTHwGJ8`|dAF$x#K%|fWEXj;9PqQ-B8dnmm}zDNvcbSG$_8Pk;_ z`JxcpsAF(cdM*wz0FwV6_fD0Gez*;j()npG7*ta2o70mi=-t8IGrHf<{s!aa=dO|c zVB!Yg^8tC$fnRX?oCtTgJIb&ppFZy-f#M7T^9!t7z_j6evCw=DsO(q}eiG_v!(sQ) z+h*#tgCf6VObsakn_nH=^Kpju_DsBU0l!>p@{>a6r6g-1#xeizv;4zi28t#>#-HD) z2%ON!f996?Ep>YMp#f7kx~i~akja$L9&-gAz^WQlEND?U{EE*+s3TBB2zZi2z=_GT z7orLFf?$uT%9;q;VOh6GZT!_zEx-iYjN;35l;t5mCW@rhw#bF;V9R{ zkKeC5(q|eehAwTjI(@c&@=?lC=o0jO90FPfr4*YZm@<12wW>obd-Z0>X@SKtTPHqL+l+WM{8NV+49e><8Ad z+)80aBamMcSCxYHLfc{b;?~nsAJYW5D-BfTS>~qt9B#E#Y2iGYU>LI}pU=eF_njcL zb2G-y(B{XajWoQoebA39ykc1kPImZz*to00P1j1X%HNyW+A>ynfqfhrC3?>%8{>XI zG~{E3&M%i&y8POsYJxJ>NvSimG!HV^J`=X^F#*?48I52w=SOLuM0S=7Q=VO``qX{@ z#sJD_!6d(yP=qp2#ESBMvky0K?&H+*^z2WO#QQDKQQ61 zSE|V^mRn1*67{>#>F6&qK)%yS3yakNDIDNeyr5R1)$%O`ZQvZiMeZf>*GNqj~6_97z(Xw^P19Sf~`HOHz1)`^; zIdM*j>o0K5sc<-4-hWu!crKsNI_xy3WjdC2hyG!$l zzpG@Qe2)_t9tC1#B}qJQPnCHjvodneQ9pj)s|tmX@*FGjHe)+!mpyg4938#pVPKv? zgi9G#>4XIT;GYRse7Tort+LZWNlmPI+_Gbk_?|g7j4x)b&hzC1bkj8@9}&`YAFf&& zz82O0(+K-^4#5CvP`urF7}@6Rz-$d zkU%nwLWY&#`Lq7#mjED+u0fB^|5wF$XZP|>@!_W!d4t4cmwHF0>J(&6*%wkdJRdp5 zmu7W#ww*v zR_)%)263xf;~I>t^9RtsKvYU>ipPbE_FLmjh)oYud$0j}Yq_oLAn@waCF)0^W``6{u%81_R`WnY)JeD0w)9B=iThaRO07p9LDxm*jBmXh%$lXw z33zrl$f!kr*4UtMYSaR;~#i3V|uZu6Yd_tOzi*oD~aLBLFamOBZBAHVKG4ZP);8 z*o-q&Y7zwrRW#`eMqUKeI!IgaWGsra^4t5lfsec^az5kCj3l7mCd-VVm1V(7P`CPm z;;MAmXu5U+%0hi7xu*4Kl>oFm4kNFcO%8SOSXdcUK&?ko&ei9kihNY;7<3o>@yK-q zTT^O+UWW7}~{#E(P32D+&@UB$t>1H^NAIv30!PK!k%2`@>6$)wB2x|$1?5oxjRbWAJR zJOJy?p(Zl!tw*|y^GC~uPP|%moVa+>qu@RzZJ(|~fi=Epxv(FIs|~Ch-mIK|{EgwK z3p@B?O-0-Ez~OrJ2-vJ^+wDAr8KwF9=VqRJ;M1eQS;(bQMX^}^+n!4~bwll3F zKq09$;EE8pGlnT}PC6I%O4D2N`!vPW=O?r@6VGX@#y#Sa4h&WgA3Q##9qg=yH96OS z(%Q~mQ+(7$M-OoPqE)`~6Ge7*SJn&RnRK^ny0Wa@E1HD+J?RbAdJk1d6Dc)SuY;;b`M=oywkqd2+@y;CB@Pp2!Is#{mL<=*@B#qrJuXZA+3kgVFZcb+$eP_puH$b(v( za=F_+5$_i@S0Kpn{^{@YHA{1Kj1^I@#9oq)tGmVo_FTEU2NnOCo%_4Ph;;+m&9Rce zD=92=O1UFlHL!&#H#$;TSx#RE<{?VDZ#896!1nS<1L0MRuaAfWvt$SD8g;k>(nR|Kd zoq}oc?Cd@q8Jjko-RJcpE)^D$x#%wEqV8yP69;tEKO0f+l*up|#v^nao7ir>*-(2+ zIK%Y4@v`1k33#e6my6LS8;%QnHp7=06)Z$kME3gs0=49C{_J_res{UWzN(-=aYDMr zl>MQeflBtGgxZPW$Qht5m5HPqg%CubD(BeLJ?zV-I{b3{KpX`;f$Uf+BreO&l zvCf})9(RJr)3Vipmnn4Ya}*6pTWo$Ek~*n$X}i{K;dqM!6CLZ4YFC>#AZTa2DnTEA z=P|bIRUlelDE|r$(S!qzC^j47H1~aNu?FC~g^uRnC87Kv#;8e~7^@I@&sry`jXv{d z_oJX+wO5;P9bK83dQoY2`rq?H|}lUiSU_OOg9)R9y=Ko5cuN9@eQTUCNG4(iqE zPPFQCCcp2`rH08<Y{~Kp1+ki8kSHS{=W4u&}?N1db@0mI=o{V{~(q*eO$`+ zs=1}kXRcz0xZ1T;cAE1Te!+a5jTY8&-AhR%eY$y*6J})o z>?RE&9l9e5GqyCpn3bt+bo}fWX+2|H5bqnDTEfc1ahEJiM@)siCR2HsvE8BJ!{=0>F6*oqIC?-A>Ckr${}x;aBYB)$0Y(my^Xztu0cysP7Ll>EJ#i$0Zv zZx4$S@Ae`d%snLF`0YbhgY>KJL?)TKHZwyq6P*U6GjXZuiZ&gO%${FA>H2-Jhxiw` z_NQ#O=F1(+a~2;)QoR>dqri7n4LJ0Y>1mVBR?-&-I=L-P6bt@&c(9-0BPomiAa*l4 z15`MMPlSPKC?j>b^MbEAIdwFn-oUTdOpmLow=7>@3yQsBF|&205gz9x*5_)Z@F`Qn zV}qcUz#tZvDe{3pk7=8}C-~+s@YL}WOzO;8!M+)0u^yp7L0a|37b;TH%UyqdH`u%E zMPuAod-Y)fe>W?Wn|?tMv3pg9)G6i!@9*Cxo))e=`7IPGJKksLjD%Q8spEtS-Bx{D zK8>%`&MN!`Ub=@1voi;s5Nm})us_02-u;|-!-{Z-`U|85% zX*gS3+un(JESi{VrjD2@#7u=!Gck|tAxI@fjn(651*N3atVIz+3^g?lik|k+iWnL- zrskSk9XN(M)jIlaec$W*@m<&2YwvaKpU+y)TI;^==gx=Js@`&-*6^;bB_FzW)3@3) zjaIXBZlw~>?%-INGb1)S8WHoZWh$)J!Eayg)$6^c09W-FsyhsOe*mo$GU3tDVQ;Ve z0A}#Kul`QXlkCy^P@7dz^VRbBvV-BdgeD|heDv+d({oUP!+X71#>xxnb1T0kuBs_2 zK@WjiJA}~8 z55vzdTK_I%Hi!ArsM5d$((W05h{2Ht#;2r)76o9*GR^Tm^LD9N{ueg_7E9A(1De&3 zm;)bOEU*Jb{#kK>)0MH(&ko(v3tYB91oMtDqHk?&LIBbG(Rz}95^`%ON@B>d#XE`r zqA>!6dTl!KPAyFTvnxv9)<@lu(EFa@Ue$i~H?6%DN|#`6tcG+Zig!)DDlk7OYTGAj zW>(M7@t5Zx#;h1srHo6Rt*NTZb31aSgH&5>F+V;H1%_oe@qeAg^OLXVi9)y&lLA~7 zNJ(fpg^@Y(oVLiENLvzzKhl8)@$am~CQgSM!q{%pz%CVpm7p&p9Dn|Rpgqwlw)W%I z&m#(_%btOX8xP4;d`EhOW~xzabI02Y?H#u{hp%n*?O~?JFBazE`5L*M%qBjo*`&;` z{YyYkcZ4n=Mgfv?j%uOHZKJ8HT~?wn9WK$hzJ9$2h_jg&8q6skJ|BJ}pGq zbtWuI+cDn{W?NhTQ&yLE1x*zZvGF0VG0Rg#-d*{8j{qhh)LE)DA+dA!pQvq*E4@BZ zlZMd@-0+2&8Lr(bJ&B-;;lWRx`0`-9j*GoZW##1rA)bfL4ZkVe`))o!S0?Mp983Q7 zubZVC;EN2Bo$5yU*v2K9{s8ovoHaujSg}MF$Q`{e+sCNP*Cv@x@&%d(F0pNa65+p* z1LwFinz$mX{fWn}ysRyPBb*KH8~UE-B*qbPTjepQYorQq%B1Xt_V?2lzSQB;iwp0w zGh=e}Ig$)D_8XtZ7TofJ417?wZ4I7hE}pKXOyy?@GiN`Zf%o4;Z!#<>M*O>`;a%4E z{S@rAJyBS7fc!?brq_x$zvw zyIvK&=-tQi6UDQm``12=j=x&ok_fm*GG>D!*JF}tE%J}ePw&!+n$H-QCyh+h*7qmU zS6vH|LzS&1ute>IwvanMohI8<&wv;AI8ClPWm&!cI?B3;@Z4&9(Cinp0=pa5@yjJ0$Zw&CRJ7m? zOXS;;3}m-}W#?=7GvN|mkKvdzngwYVkE46YyjGU?S9SFz1`^zw{-Dg2qA#x>Et!6w zkEYuI>+LseKDPE$r3bl7XMu;3?yk0BuWjKE=88f6{3{%idSqV5(?{Kzf1KJnHP#Tf z(a-!`&@Ai4odE^e1rs6fk|vFA^^KH2dw>sy%vw_rLLOA?05gb5p!u1kkW=N$&FO8! z!uMb4P{T%|I*5p`gclJhkh zB+F~%`c0DF8#7@(1N|HTcd)(KwwMEgvi5Mlr^c#>ebt4|<(tQlkp@^4A9A`$wLwPT zkFfnkac{e(KF|LpzhI{Ny84Lom6u3bz{p&+es81S^d4fs4c>!tZVjzE{(ma4V@|`no};fprktC#y-o$R~F9@58UYhGpVxrCoMdu-C|S zw#a$umj!sGRuG#W!E@5+QRAFzna`-3K=iRxK0Qc9tI;Vr3;FHJXk0nP@AdVd{E#7< z=$v>$bVB+Ef_Ino^j7AYiv?H4X;C8)UzX|pW*pu5-&yl_DVtw^01uDh{AX)^ea^gp zRc2Kl&w527You%V^VocAS*ddF>42~rAqU28^24Jl7HywrUaZ8VNE7Z1331j-n9kWa z{sJh-wTSu6r0ZWceuj`OstO@SSW*02z2eQzFzD-fQ$+o)BUbiaE&6&AMKsg z4@DZ+S<+u|ED)$rdrsuj!!(YeciT5lwJ;y`+vfprvy3HP-iPBD8O8|}0Uot~c0AVT z|KEzuKqsFi7l|D-Ts^Qy8$PNa4!oH3QJ8$9A&lbf3)yJ4WuNG&W@A@4YIyTzyh;11 z@*0WV0^*sbr!bYKCUs%_E}x;m<;%V(L)YxuW`n8Wk2-f8bDfU#Dr6$HLK@@Wy=qE) ztL-{(`|qlwlVaxg2cMVQXQbxewZy$UdSkrlDRGw}(+_!+lN!K43NNv(OVaFvANV1V z)xiSpMkK{hYD}GKMWkEzq?V2YpYz#l%{XyQ(?<&azAl3ds603udvLS2lprPeuVl`H zi5;}y=Xy>uC)H7oC{HsY0a+k@BJj$Eiaw1i>aL+%3?fSE9&cIO@u7Wh>Z15p>$R-d zK#&C1)`N$&0n`pUAE|7FtuScs)L_}oO&wuW+wj$f1CL1Oh2OtTeJsTp-e46bi8~~- zq}0!A8}N9LnA*5@w zB~d?MtZV>KRm-q-VdR)PuAwYYjw0DnEKZaXh^^*B?@Ixapl?8_=KG)m&sec$(rZ3E z*B6UpS0SuLg@7p|Islrzb;V~BS-UuRbk#~+RYX{n37ajVcQM4s9JoAGpE@4&wy+1G zqLO03+iny(jw4F`Zy2MC0>HFjP;&D3VT>M`twu~BCPq4q%psoM>?euKNaz9eY=?E; z2ct1vw@(I|R(p0gZF=Jt%}kcCi3RIVUK2c~+IpG0=Xyrju2I+4NIC5cM8f)}hz>gMZox*(gD@tW7C2##u-Nc=IyMWwB$( z&gX$%AUs(N8C0qvs)!sbSsV(9G*w~iS*aoi=5$%Z6W1grnbZou*8`IE8V2^7USWdP zL;Z1ilg!@<6qse$pZg5-r|Saqmf|TKN+JxS zBqldux{+AgRQSvvp24H~H3lsj`ip3@RxTbgCw+SJC2{=8m`a9Q4_Wm(2TkW@<%44& z!FvtQPxp*>yEZ%P@|CoH#|BDwh<7i48T|Hc*5A#3;!yfYOq`6>I97iF`R6Vw^KQz$ zw68TT<4hA;rk}H9PG`Jhbet*Y5d-TK-m2r6ww+}G20!83muwE0uWIIn6+ZBuAbzTn z`xTVfdag9q$z*pYtCIAZ2Eoo<^-ee0Iau9Hbftn~E(6NjPN-i zhLU+zO0{LhXknLq(qS7!y{j(-)8LePFS#&Td<}=LJQN=WV_&JuRqJ!H2lP7d`F(0< zfΞ{61I_n8m;>hk^X`7lP%3`y-dNsoB`cE_o76FgqxTw&eqD-0VgOc^OygW16+O zU%~lQin8da<_+~)QSNuJ@4<<3lq?tQ3y(-$iOJb`f2;b_20ZWqxOVmu2FKt`Wi!@; zn`h?LDCH1g4`8)f%2DIFE0=l_ot^a;>_xKKOq>9yqql{`r}q$)ukI^Cc9CZtrB~ZL zryOL26fqE_AdH5u)#58{2NgU62>taqs&L7-fZ?D{M|X_68br+@`C@+|S+lF756HL+|H# zutr0iunngA9g4azxm>tln=<<3~4ylyob!4iX^qVtdqGj zg)buAI|4CU`b~$M9wZFgsXz9{TMna}*|eXCZF(?G21GwBo@Dv03?|{@7_=W+_ClnYABMUA?5L2{9(|+;_ z-2V+B25CxhXiK#hxNgK<#^E$M9wfI=tr-~Mt`;UBh;d_Np65`&HRAUga-I~McZ7Lc zwp-YSaXHyB3OS(nC0qnq=_K2fLxSg6d6WjmY?pxW3c!MzoJTnq&yy8o3Rfzp)bFk- zz4!qb0R8##{lzoCDAt)l7`laeus)-pdu+BstYRpLoFWJ63tJ|Dvdk-{X7FNIp+-^8 zPv&WW*fD7b@y2(_RNL}f-_0A}uGmT>?M>G;tK`zRT`LX=<= zBVVZR0)a=+rcDh!O(5y?*2ue^QE@Umg!MU1VPh|}P0ba|-yvNv`i@B)OIPe@w++a> zzEO`vMTVTB_;y|D64mxlq;b11?bpbH+NQ3bJfzmoBk0edoh-nUxcv+w=Z~5H0Qn&t A1ONa4 literal 0 HcmV?d00001 diff --git a/models/Reference/segmind--SegMoE-SD-4x2-v0.jpg b/models/Reference/segmind--SegMoE-SD-4x2-v0.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7d7989ec2a1c0825b092263fe09324ed9a995119 GIT binary patch literal 21996 zcmbTdc|4Tg|37@qFvw0r_MOI>Wyo&qL)Njx&|+tjvQ|nBvNYDQ#Uu^JFt#YW6xr7- zLy}6eE4>RzO25nd^Z9=7$K&_cecu=7T<2VKjO+Ehp68s`cKx0Gy8v)oTUc2D5C{ag z2>t+nKLWny*Ms~4z}6N}004jmV1x((Ft7yyiy=h#f7=!i1ifbcjsZUh7V|{%6`~w1maN!Y=QP-~D zh>pLTkeHNwkC2s}lbd%xzkpOyN-3k3SI{b-JZ)%fYHn$5d)eL7+t)uZI5as$pPqR= z`)2Or;-{tMmDRQNjc?z#cXog5{oMa`Bo_pf^M8*22=;%;#RJL(Wn^SvWImD$0=)(n z1|CM3q#6^iu_Lor=y56aSQfYmvADkLn6w7wA3pD}308g?JtKXT0h>Q;^of>?6Ivc3MKUKJbDm^nvZfMkf&W${4#{kJDvf+B zXUh1fa-I~_C3%y8hqh|AfSZ#ejm*YM??vJtVoRM+DUeE4I=)V*gab7Usg$6z)CrYx zppGY2bUR30o#w_}wEMTuR1xpM__or3&T?Oqe?x__wW8Vas_Z+*kdg&7wnQ1#pI4FM zz>wnXM*|k8d2mefxYR~KQ7E-hGG5bS6vZHvM#3{`3Tvvx3wa@d)S%Qrh`^M# z+{Q$WnvLS{LEEYd^^y8w!4@6pqaZtj$m0wgOtI`pApmSb%imM%J5c=h%h)C<#dHU? z8e=lcTq0HKL>bOE(?D^WC4q*`Pac)0N@*CAnI=%I2zVm8L0K}ri~_}s1M)Cc8F(CC zhAGp55kJXlD(i&i01+c`05((-6))>#>xB00JVMO*Kd`~a)g!^1fczIb3l0e-2FFD5 zv2JIt3h32o0UYW-&?JJ;!0;ecK%G!A#}S|sbci%_NFZc)r>v;Z z{7DW+rhjpyK~!rL~voSlU6s6-IMVhwZ z<=#1Lk&iVy>s3H8)5mZ?W}^uYP3Sc?X8t+jjF+wHEUDu`_>x$(&rB}PLK{P1 zl2w+Fg0q%bvKpfXVG`tH|0&2sG=~Hd@*fbVj46!dBW@W}<3TvViZ|YdDK;zKmI*J* zMBGS;|a&w;U>=AFmx9hmMEarPZ0rJWa++ zJi-D|8!yDt^hWD%Z)vo{v5RepkeaEFi{pr~*P=1&-|NFm+5OtsUpp}+u)FE#^dt8L zS|^5hEwP&?%$hluQnz+XiAlOTN~HC~SRq9qJiAxLRb*}D^jW*-Vt0p9-!VIG+UJ4; zCa%akytO-4JEJGy<4|k?GqgArm~OuF<_yAIf46q9#9XkmG*f5n^nuEid_Emf#Ttd! z`dI6(k*gVuRVVWtw#F2j4)m4YHl-4j41EY~4O#E_9&|*h&vt1rNX0SROk8a3_k=11 zcZvL7{R@cIb~L=bA!ud%J>~-r7M|A76tp^P46u_vPuV7_j}T4^DT(y+&U{@NfZwV> zk4Ao9TbJO3ktipYKJt9-5>utLsd= zMzNCX4FJA1eynYJ{)={-R+08FTfJ*1AJo6J^b1$aNk1l-5MN;#qn?J(C$ycgLR6x` z-CmOh;wQbA<9Y>snUfF>c&Diq3MLCE?2;oX>YvXclFH7=A9s*vH?~hAwbz%ycp)`^NnbAFm%Ri>RY5tcyxO86goe|7f7tNX?uDH+8qt-wo{c zA=q-m&;AAKHQDdg6 zTdAHavPE&Ofy*M0*ZQx`bVOVbQa9dU6s+5L0?Bw4_o~r{mjm~uH2Z{1kdhe6(pfF+dbS~4ElAyzf8sJJ4rlHU_*DtX7bC%b^WEN zvXTCw;Oe1E$7+ZAOdfVljEIz@SPV85`gVrz8d%n7N`{f%R8F2#h#fWba=Vqq@zFLw zb1}J~?WqXsgY@p&Et|kCrmI(h4O~tbQuSG?DSE^@{*UM7%WlEJjSvL{1Y+@+kBxyBt)17MJ6hBjU z?X=Yi_JkHbc3V=`XFE>4$;w#zxRhHVZJ^9~6w|8Gn_BZyV4j`m=qtHb18*-FY{>9{yv$&zI3o)u8pvY-Ct%OMI6SQL?m*|=v5fibA+C=-! z>awDBou3eW|HDk^(Sj+3bER4_Om<*+*f8LNc|=TGYdS=bJPv?bjXpT?IvE3$;1vkQ zuIP^_3PS=kHUrddKyFI@_YCM`Sb^&xFRvrP>l{c2wS`IW0&)Vy%SsxXyfk>Lg}e38 z8KRkxy0^{p(eqdRnqt$GlGl_{CtH&Aiosxc$kKL{)D%+hCgTUm50V@9J2VxQGsm+r zt9*5InLTBza1kF?n9muVFQfjOhKYpOsvi?y+~k&X=m$Y3KJ!c{ztrwVNY(Kkh}vqp zXRV1cUVY0nPaywtmAa6pXS~wja%DWpytdrZUG+TrnRbnHt@lSC+a;Gx_opN7*O6Ms z`2E_Pg`f7{33bc3I`Bbl|Bk-%e65p-$J%V|V2=2YSA4NITQA6P)VvbqY9U9R_Uqs| z{q@9EqrN1*;kfQ{DSZ92wk+#wHy-nAOQf#Uk7xIV5>Y2=Wdkx*vrcyn_zC>p$YOug zv5$dnj!jlD!VF70%=U$9;I(`o4^JqYMXdHsbn%4IL+xpQBYoK!mI31?5X(xl8Y|+b4-Kp^PS`lfE9<%#AsrE#Y zC~R0liJH@(y9Q5dIcKS*rgLa5IymQQYi4VnMVwzaRa|+tS&&h!7#DPj=SB}t`_?Zy zdgz7|aqUv%@nbeq$-vM^Sb57AN9+Lf%lhX_X2g_i%}LCO+KCO0@ z*T0;LGxole_-?Y~z&9-8eNXux%D(+i2MGLAP~2@F1Fi_elKerDq&@YJzkpw&@L%Au z4a;);U}jwCJkNNBqm#w8Q^ZrlfBKAG-gz0zU5f7}_Sa~cUtdzNm=QGx+&CJxn>Q4= zG|CM+0tGMoN$iMwy7&&CX2PG6Nu5t*o^=!w(?~7a2Q}-uF6s}7KL}_ zMs&1U4LgF8g9NpAHZhmy4D7c{y2A7?`jVo$r`!Gl3Q8xqeGf2`0uXamo<8$aSd?q7 zdby^KBYj*ie@`trv+!15&3u2mk|i|Tyw5^mVCaJmb)I20M4xrAXCt*1d%TIc48=8^ zJz1VZ9ZeH6S+V#Gv!tj$(0LTagj=i*ceysc-1sAT1H;l5zbzj9xKjUAR3B%-(ywW0 z^)gwt6ssc>58oVeozlFH88WX&Q5-g`K~r&xyRVp*e6Ipmf+m6Xp@uQ_NC^d!CrN-Y zilVj%s+Yac%+5&?$`Q^w9=pcI4fCwSw}(g-5B181jg~b5i;b5T)_Z-I0!wt~*x45w zE+(bto)%4Z75PyEkDp74MR^>!=Eu)v3N9)|)iA)+1YYy+zv3%b(vK($E$n#rm@(D^ z#70aZC&4FAg4R`af2LRXDqm&pAg)(6NIp4I7Ai)~6au#N<^BR|*SGgFwjO<~zQT5{ z7xu=AA>{Vmh~hxG^tjzgJu2g3@oRiqR`Av9nmIGn9TKi{JH(Tz`kvhWHTh}}UHCDT z;XZX?O^cxOW=5Kd{JE(I?eWr|B!p;2yoHC6&!kc}tV&H_Dg~mJp)QvT#^W-G*uomp zgiUCZ^U@veikVMWyEy|To=h!7E>@g539g!QSq}X*4(j1uNS2nzhvStK{66C`fza=& zuNZ}bgHf9{Y~PsgFH2veMrZVq(ib@f_NAX*qgwHOP6`>+CK|3A4Lyw%ex<{C+x2{Z z1=%NSL652{5@)cHnAhi#$z}WI)r&HIhVrIl##f8Cb0#F&64vpdKey3rLwY`h%cb{N z2A3`T`kr;sv!gV73Z;66x(UxGk`3S=?*Ia#C#G>9~@`Gx~t`k_)%o z2u*BP;jyEQg-Gw`m8g&;?UOSNjlzB;UY0U%`s3CMw(ah~;481KJ7Y!){Zleo@4rk8 zGVAlqIwVd^Zvnf4yOmkz`r&6D-ItCn6!MQ!4!hGct#|XMoVFD)gL;F?ulz|=Ro<8} z01iyb&r;MH8t^QOdHKn5DCgBO>u#yo!kH2Z?~<3L^*FtzyI6+*2Q6?$y*_rOVZfx& z$tTY5VZm$fD0p??a@dEYt!bf4LFraHf?u6^PwEz4;r=?xbea7dqoK=tSA&XZ9@pP? zZu_R<2+BR3U-I~d9Ck+(na%Z&yz(zlMF!&z#VST|TsFT(xZ$Y1m=Iami=imUTnljmh$Jux=KuE3Q|S^iahv!lp4ML7re^nBGg?Ygtt}_>xu1&c{SKlmER9RiWp>AIH}cl^ z!2ny$DTdhX+kB5=!n|J=uaM62je2-cT`m_Mr-v*Y^1YIsuY9v@|H;KW#FJjGt|Kk6 z$K2?g_vzL0^S$UOwmf7?Jo^~eJFkHqgEXf&+pIr0B>r{IT^Xuhmx| z)<4hQH!dRR>{PnmSCV06+0VJ7fQqQvmW%dva@;ddPZA%>PaJ9W?+>xHIiX;u^A#O* zFGTgMew=*W)e*|hx(U+-<34$5ZINjycXa~ftFsm6cW!dgyEXc&UpfK@odg?7Sk%+T z-ftlf>=YD%@ppqQVaD2fl)xX7uOscXd>8}Dnd>xPT^Kn3rBk1QKuK=(R(vCQKdf2e z8{W7Q?Z%$|sEnVji1@MN!hs1#`km{g+qD5NCxbh=)CbAnLq3|4aUTm)E&)^jYOU$9ayQzN zA=abLOyWw8H?>#Cv%F&M+erHg`_IpcQS$NZs^z}6Z&q{*)p~W@N_`pV-ryc{oaOdd zulM`aam;Mc`~q!6wV%KwQ4*VX`4`3*T=IJOb+N_}@G6CA-Y6*%N|MUs8;<&maWm z*~KMjnjL2i)HIQyuH>@@Q6?cZ=)TMWU$_p!}PoOC0!ujSk7UA>FT<<`PbPw)-2wGas}iJ(c_2 z?t~7~oafT+qY3e(uhWzCtOhi54<3eE&k1_HyKy1k z<-tv#!iDKTQy*{RQ`%is!<=&}s9qb^8v=01@s&qUZ!Ap7l*`57MrKB6?Sk+g|ECvs zE4G;WUg0jfr#)#j^9){0Vg01SSN5=eJkl_(aMF7^(M-WhRX%^WvaZ7 ziKwxX$;(SIidth~?G`t;QEuzfS9`e5H*!>kw(KGY;VN(JYjlS8+baaUHF{Yrbf=EX zaS>9#Yf~kg*;eH~V9oTu$YlnHwG}1`rYJo8c@l7KsxROFbanswetsl0ZmLaxx@%jY znXz!@{o~+06W(OT3EEljJ2T;3zZPqkkOK{(7DGLr%DdzickTqS#HY3m1HGQGzd$ri z9orgiRoHvo{kp)YF;44DgUD)?4e79{(9Bwz@ zk4$|bVwxQC%rU{G33m41tMEjXV$B?_p7|%R{%@}0_mKOb2dI$cFVId9kj|f8d78KM z0QvG)tg~p8j^_@Nz#EGiR{dXIrj%38a&x4@9GUF-CtZJr4blCEGlx>crIhfXDeH(v0N@aIa`YS&C^%ZQxZ~O z8?9L>3HQ#pgv%D#H()HCMSAAg-y zIu3d5EM>TPtTSInpCCnqOYy zdQ@L?2zUAs2%ppDn+j<1ej+LG^_)tT+!ln7!>v(n>^!>O$um+@7A_+|$s<9L!A3@h zZC(PuTqK3uO*yxU8qFzx0ZZ*nMW6WD0~cPujR&R=pScLr-18H~bCLIgYq>^kt0Z#Y z3THQk$=_YAVGs&3`86&Hj2h88OBT-6#}*k&tZ73ZuYbD{3boXH;D3*Iq3Vlt#lqA> znJB16{I5wtDGQIsWu8sR&$@&Y8Hkga98JQWm}7jW1mXX1I0%pH%$7b^D7=O%uVJV! zpQ;Jae1Bt9F(YUC?A!4f)U)7Kq^FMUPhXC%>WV9|(uJCOS&Nq%u3vsxl6Xh^$tvRx z)k>O64Q-(7L^60b#WTD~y4iVqB1A3S&4;l5UbaYG7@k|!0ct03AALKJm|UOgc2t4&Go$%K4}1KT{5!CL%-$W>1Y{TgQ55IS{_Bm|8nu* zH^ij99n&$OQBBReCo>PGc~vLXC~92TAY}eaXJrl7Z267+q8PFqH3CPq+dQd#OAj18j({u`&`E`dwcO1QwPY4iK< z@kqv8n=@ZgC+_At`n`%~JjZ+7#jF8x=|<)AYaM|5Hm=mFFAzGBM-z-$6AT=59(le4 z@uO5GVu6C9DnWVtx-jxx`er-+#e1Jv8*}%{E;QZOQfBq-vi&O>HR0++8TXjGl)7~^ zApfKJ^r!^km}H~fM|i+*+rZSNDAy|*Jwkpg_Bg*}HfUV$1ChLqOU}N->Z&d>Q~t}g z9QOot@5au45unh|GG;3BUp_8$n|iNp5>bi}TzPo%Jj{Wx8e2qPL_pKtdRbX88-%_u zh~{2p4%(|UQTLOnAC)1PM$aUVrhc{1upx$K;Oxu+c_jO48H3Ur;}!HV5!PnkUZ-%g zPR_;RCzHq$7Z<_KiY&i_k zUa?xk=&_66^J)4Srta5H|7<~<`m{UA%3WE{QY(bs5Zw?}-6{|fx|P~%p6dWYHhh&UR1%d%9j0$o+Xndq0nXjINDpg#?qWL zIJ}?GWnldm*xf9i4qnZhkH6XzeR<)~B(^lutuD;Uu${KT`-CrGO!v>%UPsrh_H9#6tTY;*LV(S`5;`e%f`J_76>D~mSQ<<*{r03+-=?7}*-T5H> zTfW3Sf|$;UW9ChNB0g{@K`+~D#2j_)&MS-G35x06+oNhO_YPLS!|}JiTfx1ucPsa% zl1tx6GK&2{2x&xCJ#bg<9x}Xr*F|``>D|E_S#twU2Cj9qzrs94X{cUNa=opl=>cQ^ z-dQ5g;f!;}tKWfw`F3uFJ_eN|bC>5CLX+F>#29-0a!{))3X zQA|M+GpBjgDsZ6t1YF${<88pO9aRzxMZx}~tk83g0+$&S3RB5&?99`I{tip|vzlh_ zck@*Y)|S~wFRw(8oDkayOJ8ZT?Z=L$F9g&m!bUIXy{8=uf1jIpZaZeF13^_s&%}0_Ra|6cKfE#t&$Z@+4dWp=Z3|jhqkfl z4lAb`yTz*QKRKVS|7xG(+4raEF-KWjS%XgDpOq`vc$=rq3Fb58?kg^)f9|{_$v&*j z*3UMn?a|;FdmsO;5*t$Sx?1{4itm}OD1qMzSjh*U)fzo3?<*JU9VPW*tJsbwF8HU!5s*s53JX&g{T;IulrDvxu?kx~ z+2PuBHpt@&1JbS5hfElnS-785ApvR@ih*vitv`QIZGVr(sKG%mjLU*^UJ}k*lbko| zHs+DtYb?}i+cTx%H&P*2%2R7$7nFQKTZ{)3=URb?&tyK~e%e(jrKAvM9h<)Z)}#IB z1nYv<^_N9=q^;t#50Xe$ZY{SCC2p5AEqb&pQ8sQs%}p4+^WV@=irfY8E zrQXUL{U@Tn*%_ecKIB)g*(38Oo=7%6Iy6{OJH02j{uhY3ljdaFiymBU&U@E1ln_4t z!^qS=am#ILSVljf(nur9&$Z}wf#Y@IDFb7-LVJiQUMFmSU;)kvlMTggYCvRmpt$k{v7 zY5j7K!`L4jTz3ce|CHD+C8fvb<-d?5QESM$yPcpcvEPEBXQm@u(*syY*Fu=J>IgYBzH%$=< z`4!ASBvp!L7+0?5`ws%jz2seV=#0wKkuG##mjVKI9cD5jAA!k z;P1;7Y#B~_wFCp^{-%*&*t(FF!Ku2wH4KcUJ@m%n%pgVDWSh-AcMkx7`E}85k&nOL zpOTq(2~TbGD?1Q_?+P-d5JqMck~NAh$%|b|>=w`2h1c>c&nwt1@G+(@MNs18QoUOD z4A$A58(6~!UX|MqmCpl(loK9P}^5@(@B zQTY<~R)6+p`isNLy0$jy&yOVT+!R#^aY;364Lb~Zq@h$bqkiKrusQw5(z^K>ba8r8 z&|YDEPk8gsl-(LTEt2i#6Z755yu)7{sYZQO@4IyC&$NtOWBc$l0~-bLow`<>*Js~g zp-P-(o@$-%FsE5oK_L6pKfJYAcR1H#5Z&x)Pm?I?7R7<5*n1H)|)1gsX+zRa<$w$j?aDYC>GGl?Kg4E&)L_;Deky7FXCh!E{|)Ct*dmL; zi3P23ck6iLKVSc{a8U6NU7UX4!02GC+*1WtYsgaTbP<$mT4+g}d_($hi~mK$=wCp& zmWzD2c6ffL7M3E)`F=wGkNKp@qZ_AhZyD`ngnw4#{c(7y4^l@wIiGfW<1cV;kIK~h zHJNC22C?U})t;*G=fv+z56R{QS5CI%3?kE{i$3Gvuw=lO#1s-p%53^GXL8=v0QWxI zUzuA&ZhtmFH0S04hJp5kWSNSj`2KRaq$L}x-12MZcLK(@A@G$zgiI5K4R5O3f3VBM^dEuG8g^L1pt`Q!Qy zoP9Jfe`z4<6nWD=b7__QSw8lJz{}d{ovqu{oUoV*$cr+4dH>GN!#KQ9rP~*V@^$Mi zmf1&FuO04n0LY4yY(KN@K3W**0)aNH*WPQBnRw537$gS89@ixfY?}5GZOnLWPB3yd zWI(=-sw$2m(P+r%%wC#GUe%)??zo~e;Wr4Lm7_RoNSMnZ5$&G5tjy9iC zSbS*VRa=JE%gCQtwSKt<<)w_lPjJ>{=gATop!{1+3v2Eg;)Gg0vVrlUm`=?#Ul9T< ztfrBikqI$S2zbcswC^CfR?Ot_7npV~Ya6>!^3nUire)LJVwLWAwf?qMqB`{{`+>(y zrd#`u)Hn_I&!20)p!i29(~-l4PhMgfne3`T!y>!%ECBG$s9#($tjA#O? zqgj8bvUW7f-zAcLy>Gi@rlKjgY7Ihv_X?5KO7y1>Hj!x~GYiSv+b7Q@{mL*!aWy}} z-3U2taWmSPr%1uVJfwvr;!llU+)*_7Oo&BW=?S@ z5`d!|Cor|EpA}4`QI?HgCa zfX|Ypw1kz>eYw;>!_4~sc>a2{riWoTf9(90jfHh@Vb<;KgGc{RCVc`V81^v?VQ)+Tn2euUEPQ9v zmSrY>%Ri5+MbXSux-AW>L9~CgHR)q1Pn*lt7U68E!WLP{d~??@3r~~_=)3)#e$Wb) zPmR>C2d$Fa*;G89I37*%v6e!8==-$%L+nMG8drEjFtgEnLuyquevju?AQT zsd(`Z@AJj*GNL#Q3FaEWhQoNZc76q~%q~RxfsYnLKqpYR$8()J?`)2dl%gu~cKA*T zF5=<6Q|S#e&lxhAYg9!f40DgsWehlBfSZT^Sg4Eb=bGOZcw`BOfHd7T%yU4{>Nc|+ zUMp{Xnps<#@N#>}TfHw<_~p)%!j|>X@Ryb5+q6^t9>YLV$8Wy4@4v27E zh@jmsQ{%NzHJ8dJF$VJec_$an^2;f%AOYNRsHNU@?z2B3wAE{Fe(w zYJ;O&{PYEXnxF$ljFW>wL|8g#-9|2!Jv5%+x}VhHkc=hLig_BPT|mc%kT)3DmPbI$ zB`6HNP@-8}CPv^J7!1^!`i5pnF6Xgf87K#%*>Gxp5(A2C+ABf!d1^xQ1~!6#pV=cJ zcJQ$NaYv1W3-VkHU&WnJ$MoNOv6cp3dYQ55Sn~Di^M+tX{J!hbUqGag%ddhOvxQ#M z7Pkg2!#lAs0Q;or!0d&NB5&B2FYCL=pXJwsZ?f0VGKbGUDm&9MeD?mu!)4XVQbtZZ z*T=trc=VXr4Q=SdgfK@n<%Fs^|NK$iXDzcYG-43robh&S0T&+(@S-)o62^Qx+FDP& zJZbK5d+oD=rO)m0;$ZUcptj4^$xNSYk3ATV>9R}qf(53&5gnA|ly7#Q3sNg3(J5WB zU7afrs#k~mEUX5r3o3Q?n1bBPw9WS)h!s6@nkS2LDJC@7k$9WsLyHxCK<0z;?+*ZFy5=zxQ*(mOTl8=~k zvlq3RUzC}YFGuVi<^aQ4BEBNTt_u1#2#r*kSG*0|ui%3VUiz9&v2}Ek5OV8maKUbg z|F+~=8lpB3*`s>o&P@#hb8zQ~mG=aKIMj}rLwN~Ipw0t)L(ve?XQKiA}^0GApo}m z&rP4G5^-8I#Ogo%)(~S4nt+-8_6iLT&TdJ}y|9jp!3d)G9m;({~_Z=y$ zo?o_g8P<$`{RM^+p7IVc_^?9hsnc_2XuEj^Le-t^(;s>R6>BP+{~I$mtDL8*}nKnM2?~ zgr5j{#HdoiN-~g|(2!3CbGxdc1GYY1KXYnC{NtF;vm6C&l zPDp*sVm^yJilT10D09KHzQq1Te+U9Vr-uc7dm1H3R`P7TKTJen>?{H2re&gW;OuR9 zg7>@93${86TD0J8EEdynmoM!(#eu2uE-%2ImW|~f&twXs$PGKxPM1SsU2~It{4Q6y zoC0?z-@x$3LtqjW1a67HG~V)AiM~kL#VLiKLF` zTHwGJ8`|dAF$x#K%|fWEXj;9PqQ-B8dnmm}zDNvcbSG$_8Pk;_ z`JxcpsAF(cdM*wz0FwV6_fD0Gez*;j()npG7*ta2o70mi=-t8IGrHf<{s!aa=dO|c zVB!Yg^8tC$fnRX?oCtTgJIb&ppFZy-f#M7T^9!t7z_j6evCw=DsO(q}eiG_v!(sQ) z+h*#tgCf6VObsakn_nH=^Kpju_DsBU0l!>p@{>a6r6g-1#xeizv;4zi28t#>#-HD) z2%ON!f996?Ep>YMp#f7kx~i~akja$L9&-gAz^WQlEND?U{EE*+s3TBB2zZi2z=_GT z7orLFf?$uT%9;q;VOh6GZT!_zEx-iYjN;35l;t5mCW@rhw#bF;V9R{ zkKeC5(q|eehAwTjI(@c&@=?lC=o0jO90FPfr4*YZm@<12wW>obd-Z0>X@SKtTPHqL+l+WM{8NV+49e><8Ad z+)80aBamMcSCxYHLfc{b;?~nsAJYW5D-BfTS>~qt9B#E#Y2iGYU>LI}pU=eF_njcL zb2G-y(B{XajWoQoebA39ykc1kPImZz*to00P1j1X%HNyW+A>ynfqfhrC3?>%8{>XI zG~{E3&M%i&y8POsYJxJ>NvSimG!HV^J`=X^F#*?48I52w=SOLuM0S=7Q=VO``qX{@ z#sJD_!6d(yP=qp2#ESBMvky0K?&H+*^z2WO#QQDKQQ61 zSE|V^mRn1*67{>#>F6&qK)%yS3yakNDIDNeyr5R1)$%O`ZQvZiMeZf>*GNqj~6_97z(Xw^P19Sf~`HOHz1)`^; zIdM*j>o0K5sc<-4-hWu!crKsNI_xy3WjdC2hyG!$l zzpG@Qe2)_t9tC1#B}qJQPnCHjvodneQ9pj)s|tmX@*FGjHe)+!mpyg4938#pVPKv? zgi9G#>4XIT;GYRse7Tort+LZWNlmPI+_Gbk_?|g7j4x)b&hzC1bkj8@9}&`YAFf&& zz82O0(+K-^4#5CvP`urF7}@6Rz-$d zkU%nwLWY&#`Lq7#mjED+u0fB^|5wF$XZP|>@!_W!d4t4cmwHF0>J(&6*%wkdJRdp5 zmu7W#ww*v zR_)%)263xf;~I>t^9RtsKvYU>ipPbE_FLmjh)oYud$0j}Yq_oLAn@waCF)0^W``6{u%81_R`WnY)JeD0w)9B=iThaRO07p9LDxm*jBmXh%$lXw z33zrl$f!kr*4UtMYSaR;~#i3V|uZu6Yd_tOzi*oD~aLBLFamOBZBAHVKG4ZP);8 z*o-q&Y7zwrRW#`eMqUKeI!IgaWGsra^4t5lfsec^az5kCj3l7mCd-VVm1V(7P`CPm z;;MAmXu5U+%0hi7xu*4Kl>oFm4kNFcO%8SOSXdcUK&?ko&ei9kihNY;7<3o>@yK-q zTT^O+UWW7}~{#E(P32D+&@UB$t>1H^NAIv30!PK!k%2`@>6$)wB2x|$1?5oxjRbWAJR zJOJy?p(Zl!tw*|y^GC~uPP|%moVa+>qu@RzZJ(|~fi=Epxv(FIs|~Ch-mIK|{EgwK z3p@B?O-0-Ez~OrJ2-vJ^+wDAr8KwF9=VqRJ;M1eQS;(bQMX^}^+n!4~bwll3F zKq09$;EE8pGlnT}PC6I%O4D2N`!vPW=O?r@6VGX@#y#Sa4h&WgA3Q##9qg=yH96OS z(%Q~mQ+(7$M-OoPqE)`~6Ge7*SJn&RnRK^ny0Wa@E1HD+J?RbAdJk1d6Dc)SuY;;b`M=oywkqd2+@y;CB@Pp2!Is#{mL<=*@B#qrJuXZA+3kgVFZcb+$eP_puH$b(v( za=F_+5$_i@S0Kpn{^{@YHA{1Kj1^I@#9oq)tGmVo_FTEU2NnOCo%_4Ph;;+m&9Rce zD=92=O1UFlHL!&#H#$;TSx#RE<{?VDZ#896!1nS<1L0MRuaAfWvt$SD8g;k>(nR|Kd zoq}oc?Cd@q8Jjko-RJcpE)^D$x#%wEqV8yP69;tEKO0f+l*up|#v^nao7ir>*-(2+ zIK%Y4@v`1k33#e6my6LS8;%QnHp7=06)Z$kME3gs0=49C{_J_res{UWzN(-=aYDMr zl>MQeflBtGgxZPW$Qht5m5HPqg%CubD(BeLJ?zV-I{b3{KpX`;f$Uf+BreO&l zvCf})9(RJr)3Vipmnn4Ya}*6pTWo$Ek~*n$X}i{K;dqM!6CLZ4YFC>#AZTa2DnTEA z=P|bIRUlelDE|r$(S!qzC^j47H1~aNu?FC~g^uRnC87Kv#;8e~7^@I@&sry`jXv{d z_oJX+wO5;P9bK83dQoY2`rq?H|}lUiSU_OOg9)R9y=Ko5cuN9@eQTUCNG4(iqE zPPFQCCcp2`rH08<Y{~Kp1+ki8kSHS{=W4u&}?N1db@0mI=o{V{~(q*eO$`+ zs=1}kXRcz0xZ1T;cAE1Te!+a5jTY8&-AhR%eY$y*6J})o z>?RE&9l9e5GqyCpn3bt+bo}fWX+2|H5bqnDTEfc1ahEJiM@)siCR2HsvE8BJ!{=0>F6*oqIC?-A>Ckr${}x;aBYB)$0Y(my^Xztu0cysP7Ll>EJ#i$0Zv zZx4$S@Ae`d%snLF`0YbhgY>KJL?)TKHZwyq6P*U6GjXZuiZ&gO%${FA>H2-Jhxiw` z_NQ#O=F1(+a~2;)QoR>dqri7n4LJ0Y>1mVBR?-&-I=L-P6bt@&c(9-0BPomiAa*l4 z15`MMPlSPKC?j>b^MbEAIdwFn-oUTdOpmLow=7>@3yQsBF|&205gz9x*5_)Z@F`Qn zV}qcUz#tZvDe{3pk7=8}C-~+s@YL}WOzO;8!M+)0u^yp7L0a|37b;TH%UyqdH`u%E zMPuAod-Y)fe>W?Wn|?tMv3pg9)G6i!@9*Cxo))e=`7IPGJKksLjD%Q8spEtS-Bx{D zK8>%`&MN!`Ub=@1voi;s5Nm})us_02-u;|-!-{Z-`U|85% zX*gS3+un(JESi{VrjD2@#7u=!Gck|tAxI@fjn(651*N3atVIz+3^g?lik|k+iWnL- zrskSk9XN(M)jIlaec$W*@m<&2YwvaKpU+y)TI;^==gx=Js@`&-*6^;bB_FzW)3@3) zjaIXBZlw~>?%-INGb1)S8WHoZWh$)J!Eayg)$6^c09W-FsyhsOe*mo$GU3tDVQ;Ve z0A}#Kul`QXlkCy^P@7dz^VRbBvV-BdgeD|heDv+d({oUP!+X71#>xxnb1T0kuBs_2 zK@WjiJA}~8 z55vzdTK_I%Hi!ArsM5d$((W05h{2Ht#;2r)76o9*GR^Tm^LD9N{ueg_7E9A(1De&3 zm;)bOEU*Jb{#kK>)0MH(&ko(v3tYB91oMtDqHk?&LIBbG(Rz}95^`%ON@B>d#XE`r zqA>!6dTl!KPAyFTvnxv9)<@lu(EFa@Ue$i~H?6%DN|#`6tcG+Zig!)DDlk7OYTGAj zW>(M7@t5Zx#;h1srHo6Rt*NTZb31aSgH&5>F+V;H1%_oe@qeAg^OLXVi9)y&lLA~7 zNJ(fpg^@Y(oVLiENLvzzKhl8)@$am~CQgSM!q{%pz%CVpm7p&p9Dn|Rpgqwlw)W%I z&m#(_%btOX8xP4;d`EhOW~xzabI02Y?H#u{hp%n*?O~?JFBazE`5L*M%qBjo*`&;` z{YyYkcZ4n=Mgfv?j%uOHZKJ8HT~?wn9WK$hzJ9$2h_jg&8q6skJ|BJ}pGq zbtWuI+cDn{W?NhTQ&yLE1x*zZvGF0VG0Rg#-d*{8j{qhh)LE)DA+dA!pQvq*E4@BZ zlZMd@-0+2&8Lr(bJ&B-;;lWRx`0`-9j*GoZW##1rA)bfL4ZkVe`))o!S0?Mp983Q7 zubZVC;EN2Bo$5yU*v2K9{s8ovoHaujSg}MF$Q`{e+sCNP*Cv@x@&%d(F0pNa65+p* z1LwFinz$mX{fWn}ysRyPBb*KH8~UE-B*qbPTjepQYorQq%B1Xt_V?2lzSQB;iwp0w zGh=e}Ig$)D_8XtZ7TofJ417?wZ4I7hE}pKXOyy?@GiN`Zf%o4;Z!#<>M*O>`;a%4E z{S@rAJyBS7fc!?brq_x$zvw zyIvK&=-tQi6UDQm``12=j=x&ok_fm*GG>D!*JF}tE%J}ePw&!+n$H-QCyh+h*7qmU zS6vH|LzS&1ute>IwvanMohI8<&wv;AI8ClPWm&!cI?B3;@Z4&9(Cinp0=pa5@yjJ0$Zw&CRJ7m? zOXS;;3}m-}W#?=7GvN|mkKvdzngwYVkE46YyjGU?S9SFz1`^zw{-Dg2qA#x>Et!6w zkEYuI>+LseKDPE$r3bl7XMu;3?yk0BuWjKE=88f6{3{%idSqV5(?{Kzf1KJnHP#Tf z(a-!`&@Ai4odE^e1rs6fk|vFA^^KH2dw>sy%vw_rLLOA?05gb5p!u1kkW=N$&FO8! z!uMb4P{T%|I*5p`gclJhkh zB+F~%`c0DF8#7@(1N|HTcd)(KwwMEgvi5Mlr^c#>ebt4|<(tQlkp@^4A9A`$wLwPT zkFfnkac{e(KF|LpzhI{Ny84Lom6u3bz{p&+es81S^d4fs4c>!tZVjzE{(ma4V@|`no};fprktC#y-o$R~F9@58UYhGpVxrCoMdu-C|S zw#a$umj!sGRuG#W!E@5+QRAFzna`-3K=iRxK0Qc9tI;Vr3;FHJXk0nP@AdVd{E#7< z=$v>$bVB+Ef_Ino^j7AYiv?H4X;C8)UzX|pW*pu5-&yl_DVtw^01uDh{AX)^ea^gp zRc2Kl&w527You%V^VocAS*ddF>42~rAqU28^24Jl7HywrUaZ8VNE7Z1331j-n9kWa z{sJh-wTSu6r0ZWceuj`OstO@SSW*02z2eQzFzD-fQ$+o)BUbiaE&6&AMKsg z4@DZ+S<+u|ED)$rdrsuj!!(YeciT5lwJ;y`+vfprvy3HP-iPBD8O8|}0Uot~c0AVT z|KEzuKqsFi7l|D-Ts^Qy8$PNa4!oH3QJ8$9A&lbf3)yJ4WuNG&W@A@4YIyTzyh;11 z@*0WV0^*sbr!bYKCUs%_E}x;m<;%V(L)YxuW`n8Wk2-f8bDfU#Dr6$HLK@@Wy=qE) ztL-{(`|qlwlVaxg2cMVQXQbxewZy$UdSkrlDRGw}(+_!+lN!K43NNv(OVaFvANV1V z)xiSpMkK{hYD}GKMWkEzq?V2YpYz#l%{XyQ(?<&azAl3ds603udvLS2lprPeuVl`H zi5;}y=Xy>uC)H7oC{HsY0a+k@BJj$Eiaw1i>aL+%3?fSE9&cIO@u7Wh>Z15p>$R-d zK#&C1)`N$&0n`pUAE|7FtuScs)L_}oO&wuW+wj$f1CL1Oh2OtTeJsTp-e46bi8~~- zq}0!A8}N9LnA*5@w zB~d?MtZV>KRm-q-VdR)PuAwYYjw0DnEKZaXh^^*B?@Ixapl?8_=KG)m&sec$(rZ3E z*B6UpS0SuLg@7p|Islrzb;V~BS-UuRbk#~+RYX{n37ajVcQM4s9JoAGpE@4&wy+1G zqLO03+iny(jw4F`Zy2MC0>HFjP;&D3VT>M`twu~BCPq4q%psoM>?euKNaz9eY=?E; z2ct1vw@(I|R(p0gZF=Jt%}kcCi3RIVUK2c~+IpG0=Xyrju2I+4NIC5cM8f)}hz>gMZox*(gD@tW7C2##u-Nc=IyMWwB$( z&gX$%AUs(N8C0qvs)!sbSsV(9G*w~iS*aoi=5$%Z6W1grnbZou*8`IE8V2^7USWdP zL;Z1ilg!@<6qse$pZg5-r|Saqmf|TKN+JxS zBqldux{+AgRQSvvp24H~H3lsj`ip3@RxTbgCw+SJC2{=8m`a9Q4_Wm(2TkW@<%44& z!FvtQPxp*>yEZ%P@|CoH#|BDwh<7i48T|Hc*5A#3;!yfYOq`6>I97iF`R6Vw^KQz$ zw68TT<4hA;rk}H9PG`Jhbet*Y5d-TK-m2r6ww+}G20!83muwE0uWIIn6+ZBuAbzTn z`xTVfdag9q$z*pYtCIAZ2Eoo<^-ee0Iau9Hbftn~E(6NjPN-i zhLU+zO0{LhXknLq(qS7!y{j(-)8LePFS#&Td<}=LJQN=WV_&JuRqJ!H2lP7d`F(0< zfΞ{61I_n8m;>hk^X`7lP%3`y-dNsoB`cE_o76FgqxTw&eqD-0VgOc^OygW16+O zU%~lQin8da<_+~)QSNuJ@4<<3lq?tQ3y(-$iOJb`f2;b_20ZWqxOVmu2FKt`Wi!@; zn`h?LDCH1g4`8)f%2DIFE0=l_ot^a;>_xKKOq>9yqql{`r}q$)ukI^Cc9CZtrB~ZL zryOL26fqE_AdH5u)#58{2NgU62>taqs&L7-fZ?D{M|X_68br+@`C@+|S+lF756HL+|H# zutr0iunngA9g4azxm>tln=<<3~4ylyob!4iX^qVtdqGj zg)buAI|4CU`b~$M9wZFgsXz9{TMna}*|eXCZF(?G21GwBo@Dv03?|{@7_=W+_ClnYABMUA?5L2{9(|+;_ z-2V+B25CxhXiK#hxNgK<#^E$M9wfI=tr-~Mt`;UBh;d_Np65`&HRAUga-I~McZ7Lc zwp-YSaXHyB3OS(nC0qnq=_K2fLxSg6d6WjmY?pxW3c!MzoJTnq&yy8o3Rfzp)bFk- zz4!qb0R8##{lzoCDAt)l7`laeus)-pdu+BstYRpLoFWJ63tJ|Dvdk-{X7FNIp+-^8 zPv&WW*fD7b@y2(_RNL}f-_0A}uGmT>?M>G;tK`zRT`LX=<= zBVVZR0)a=+rcDh!O(5y?*2ue^QE@Umg!MU1VPh|}P0ba|-yvNv`i@B)OIPe@w++a> zzEO`vMTVTB_;y|D64mxlq;b11?bpbH+NQ3bJfzmoBk0edoh-nUxcv+w=Z~5H0Qn&t A1ONa4 literal 0 HcmV?d00001 diff --git a/modules/control/proc/marigold/util/ensemble.py b/modules/control/proc/marigold/util/ensemble.py index ae254329d..d7a9023ca 100644 --- a/modules/control/proc/marigold/util/ensemble.py +++ b/modules/control/proc/marigold/util/ensemble.py @@ -74,7 +74,7 @@ def ensemble_depths( # objective function def closure(x): - l = len(x) # noqa + l = len(x) s = x[: int(l / 2)] t = x[int(l / 2) :] s = torch.from_numpy(s).to(dtype=dtype).to(device) @@ -102,7 +102,7 @@ def ensemble_depths( closure, x, method="BFGS", tol=tol, options={"maxiter": max_iter, "disp": False} ) x = res.x - l = len(x) # noqa + l = len(x) s = x[: int(l / 2)] t = x[int(l / 2) :] diff --git a/modules/face/instantid_model.py b/modules/face/instantid_model.py index 5ae70b1fa..7f70aff98 100644 --- a/modules/face/instantid_model.py +++ b/modules/face/instantid_model.py @@ -94,7 +94,7 @@ class PerceiverAttention(nn.Module): x = self.norm1(x) latents = self.norm2(latents) - b, l, _ = latents.shape # noqa:E741 + b, l, _ = latents.shape q = self.to_q(latents) kv_input = torch.cat((x, latents), dim=-2) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index b7ab23a2a..59fb9be9a 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -158,7 +158,9 @@ def process_diffusers(p: processing.StableDiffusionProcessing): if hasattr(model, "set_progress_bar_config"): model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba') args = {} - signature = inspect.signature(type(model).__call__) + if hasattr(model, 'pipe'): # recurse + model = model.pipe + signature = inspect.signature(type(model).__call__, follow_wrapped=True) possible = signature.parameters.keys() debug(f'Diffusers pipeline possible: {possible}') if shared.opts.diffusers_generator_device == "Unset": @@ -201,14 +203,15 @@ def process_diffusers(p: processing.StableDiffusionProcessing): if 'generator' in possible and generator is not None: args['generator'] = generator if 'output_type' in possible: - args['output_type'] = 'np' + if hasattr(model, 'vae'): + args['output_type'] = 'np' # only set latent if model has vae if 'callback_steps' in possible: args['callback_steps'] = 1 if 'callback' in possible: args['callback'] = diffusers_callback_legacy elif 'callback_on_step_end_tensor_inputs' in possible: args['callback_on_step_end'] = diffusers_callback - if 'prompt_embeds' in possible and 'negative_prompt_embeds' in possible: + if 'prompt_embeds' in possible and 'negative_prompt_embeds' in possible and hasattr(model, '_callback_tensor_inputs'): args['callback_on_step_end_tensor_inputs'] = model._callback_tensor_inputs # pylint: disable=protected-access else: args['callback_on_step_end_tensor_inputs'] = ['latents'] @@ -405,7 +408,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): desc='Base', ) update_sampler(shared.sd_model) - shared.state.sampling_steps = base_args['num_inference_steps'] + shared.state.sampling_steps = base_args.get('num_inference_steps', p.steps) p.extra_generation_params['Pipeline'] = shared.sd_model.__class__.__name__ if shared.opts.scheduler_eta is not None and shared.opts.scheduler_eta > 0 and shared.opts.scheduler_eta < 1: p.extra_generation_params["Sampler Eta"] = shared.opts.scheduler_eta diff --git a/modules/sd_models.py b/modules/sd_models.py index fc1abca30..64679973e 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -586,6 +586,10 @@ def detect_pipeline(f: str, op: str = 'model', warning=True): if shared.backend == shared.Backend.ORIGINAL: warn(f'Model detected as InstaFlow model, but attempting to load using backend=original: {op}={f} size={size} MB') guess = 'InstaFlow' + if 'SegMoE' in f: + if shared.backend == shared.Backend.ORIGINAL: + warn(f'Model detected as SegMoE model, but attempting to load using backend=original: {op}={f} size={size} MB') + guess = 'SegMoE' if 'PixArt' in f: if shared.backend == shared.Backend.ORIGINAL: warn(f'Model detected as PixArt Alpha model, but attempting to load using backend=original: {op}={f} size={size} MB') @@ -794,6 +798,14 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No except Exception as e: shared.log.error(f'Diffusers Failed loading {op}: {checkpoint_info.path} {e}') return + if model_type in ['SegMoE']: # forced pipeline + try: + from modules.segmoe.segmoe_model import SegMoEPipeline + sd_model = SegMoEPipeline(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + sd_model = sd_model.pipe # segmoe pipe does its stuff in __init__ and __call__ is the original pipeline + except Exception as e: + shared.log.error(f'Diffusers Failed loading {op}: {checkpoint_info.path} {e}') + return elif 'ONNX' in model_type: # forced pipeline sd_model = pipeline.from_pretrained(checkpoint_info.path) else: diff --git a/modules/segmoe/segmoe_model.py b/modules/segmoe/segmoe_model.py new file mode 100644 index 000000000..3dd5a8cb1 --- /dev/null +++ b/modules/segmoe/segmoe_model.py @@ -0,0 +1,1315 @@ +import gc +from collections import OrderedDict +from typing import Any, Dict, Callable +import os +from copy import deepcopy +from math import ceil +import json +import safetensors +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers import ( + DiffusionPipeline, + StableDiffusionPipeline, + StableDiffusionXLPipeline, + DDPMScheduler, + UNet2DConditionModel, +) +import tqdm +import yaml + + +def remove_all_forward_hooks(model: torch.nn.Module) -> None: + for _name, child in model._modules.items(): # pylint: disable=protected-access + if child is not None: + if hasattr(child, "_forward_hooks"): + child._forward_hooks: Dict[int, Callable] = OrderedDict() + remove_all_forward_hooks(child) + + +# Inspired from transformers.models.mixtral.modeling_mixtral.MixtralSparseMoeBlock +class SparseMoeBlock(nn.Module): + def __init__(self, config, experts): + super().__init__() + self.hidden_dim = config["hidden_size"] + self.num_experts = config["num_local_experts"] + self.top_k = config["num_experts_per_tok"] + self.out_dim = config.get("out_dim", self.hidden_dim) + + # gating + self.gate = nn.Linear(self.hidden_dim, self.num_experts, bias=False) + self.experts = nn.ModuleList([deepcopy(exp) for exp in experts]) + + def forward(self, hidden_states: torch.Tensor, scale=None) -> torch.Tensor: # pylint: disable=unused-argument + batch_size, sequence_length, f_map_sz = hidden_states.shape + hidden_states = hidden_states.view(-1, f_map_sz) + # router_logits: (batch * sequence_length, n_experts) + router_logits = self.gate(hidden_states) + _, selected_experts = torch.topk( + router_logits.sum(dim=0, keepdim=True), self.top_k, dim=1 + ) + routing_weights = F.softmax( + router_logits[:, selected_experts[0]], dim=1, dtype=torch.float + ) + + # we cast back to the input dtype + routing_weights = routing_weights.to(hidden_states.dtype) + + final_hidden_states = torch.zeros( + (batch_size * sequence_length, self.out_dim), + dtype=hidden_states.dtype, + device=hidden_states.device, + ) + + # Loop over all available experts in the model and perform the computation on each expert + for i, expert_idx in enumerate(selected_experts[0].tolist()): + expert_layer = self.experts[expert_idx] + + current_hidden_states = routing_weights[:, i].view( + batch_size * sequence_length, -1 + ) * expert_layer(hidden_states) + + # However `index_add_` only support torch tensors for indexing so we'll use + # the `top_x` tensor here. + final_hidden_states = final_hidden_states + current_hidden_states + final_hidden_states = final_hidden_states.reshape( + batch_size, sequence_length, self.out_dim + ) + return final_hidden_states + + +def getActivation(activation, name): + def hook(model, inp, output): # pylint: disable=unused-argument + activation[name] = inp + + return hook + + +class SegMoEPipeline: + def __init__(self, config_or_path, **kwargs) -> Any: + """ + Instantiates the SegMoEPipeline. SegMoEPipeline implements the Segmind Mixture of Diffusion Experts, efficiently combining Stable Diffusion and Stable Diffusion Xl models. + + Usage: + + from segmoe import SegMoEPipeline + pipeline = SegMoEPipeline(config_or_path, **kwargs) + + config_or_path: Path to Config or Directory containing SegMoE checkpoint or HF Card of SegMoE Checkpoint. + + Other Keyword Arguments: + torch_dtype: Data Type to load the pipeline in. (Default: torch.float16) + variant: Variant of the Model. (Default: fp16) + device: Device to load the model on. (Default: cuda) + Other args supported by diffusers.DiffusionPipeline are also supported. + + For more details visit https://github.com/segmind/segmoe. + """ + self.torch_dtype = kwargs.pop("torch_dtype", torch.float16) + self.use_safetensors = kwargs.pop("use_safetensors", True) + self.variant = kwargs.pop("variant", "fp16") + self.device = kwargs.pop("device", "cuda") + if os.path.isfile(config_or_path): + self.load_from_scratch(config_or_path, **kwargs) + else: + if not os.path.isdir(config_or_path): + cached_folder = DiffusionPipeline.download(config_or_path) + else: + cached_folder = config_or_path + unet = self.create_empty(cached_folder) + unet.load_state_dict( + safetensors.torch.load_file( + f"{cached_folder}/unet/diffusion_pytorch_model.safetensors" + ) + ) + self.pipe = DiffusionPipeline.from_pretrained( + cached_folder, + unet=unet, + torch_dtype=self.torch_dtype, + use_safetensors=self.use_safetensors, + ) + self.pipe.to(self.device) + self.pipe.unet.to( + device=self.device, + dtype=self.torch_dtype, + memory_format=torch.channels_last, + ) + + def to(self, *args, **kwargs): # TODO added no-op to avoid error + self.pipe.to(*args, **kwargs) + + def load_from_scratch(self, config: str, **kwargs) -> None: + # Load Config + with open(config, "r", encoding='utf8') as f: + config = yaml.load(f, Loader=yaml.SafeLoader) + self.config = config + if self.config.get("num_experts", None): + self.num_experts = self.config["num_experts"] + else: + if self.config.get("experts", None): + self.num_experts = len(self.config["experts"]) + else: + if self.config.get("loras", None): + self.num_experts = len(self.config["loras"]) + else: + self.num_experts = 1 + num_experts_per_tok = self.config.get("num_experts_per_tok", 1) + self.config["num_experts_per_tok"] = num_experts_per_tok + moe_layers = self.config.get("moe_layers", "attn") + self.config["moe_layers"] = moe_layers + # Load Base Model + if self.config["base_model"].startswith( + "https://civitai.com/api/download/models/" + ): + os.makedirs("base", exist_ok=True) + if not os.path.isfile("base/model.safetensors"): + os.system( + "wget -O " + + "base/model.safetensors" + + self.config["base_model"] + + " --content-disposition" + ) + self.config["base_model"] = "base/model.safetensors" + self.pipe = DiffusionPipeline.from_single_file( + self.config["base_model"], torch_dtype=self.torch_dtype + ) + else: + try: + self.pipe = DiffusionPipeline.from_pretrained( + self.config["base_model"], + torch_dtype=self.torch_dtype, + use_safetensors=self.use_safetensors, + variant=self.variant, + **kwargs, + ) + except Exception: + self.pipe = DiffusionPipeline.from_pretrained( + self.config["base_model"], torch_dtype=self.torch_dtype, **kwargs + ) + if self.pipe.__class__ == StableDiffusionPipeline: + self.up_idx_start = 1 + self.up_idx_end = len(self.pipe.unet.up_blocks) + self.down_idx_start = 0 + self.down_idx_end = len(self.pipe.unet.down_blocks) - 1 + elif self.pipe.__class__ == StableDiffusionXLPipeline: + self.up_idx_start = 0 + self.up_idx_end = len(self.pipe.unet.up_blocks) - 1 + self.down_idx_start = 1 + self.down_idx_end = len(self.pipe.unet.down_blocks) + self.config["up_idx_start"] = self.up_idx_start + self.config["up_idx_end"] = self.up_idx_end + self.config["down_idx_start"] = self.down_idx_start + self.config["down_idx_end"] = self.down_idx_end + + # TODO: Add Support for Scheduler Selection + self.pipe.scheduler = DDPMScheduler.from_config(self.pipe.scheduler.config) + + # Load Experts + experts = [] + positive = [] + negative = [] + if self.config.get("experts", None): + for i, exp in enumerate(self.config["experts"]): + positive.append(exp["positive_prompt"]) + negative.append(exp["negative_prompt"]) + if exp["source_model"].startswith( + "https://civitai.com/api/download/models/" + ): + try: + if not os.path.isfile(f"expert_{i}/model.safetensors"): + os.makedirs(f"expert_{i}", exist_ok=True) + if not os.path.isfile(f"expert_{i}/model.safetensors"): + os.system( + f"wget {exp['source_model']} -O " + + f"expert_{i}/model.safetensors" + + " --content-disposition" + ) + exp["source_model"] = f"expert_{i}/model.safetensors" + expert = DiffusionPipeline.from_single_file( + exp["source_model"], + ).to(self.device, self.torch_dtype) + except Exception as e: + print(f"Expert {i} {exp['source_model']} failed to load") + print("Error:", e) + else: + try: + expert = DiffusionPipeline.from_pretrained( + exp["source_model"], + torch_dtype=self.torch_dtype, + use_safetensors=self.use_safetensors, + variant=self.variant, + **kwargs, + ) + + # TODO: Add Support for Scheduler Selection + expert.scheduler = DDPMScheduler.from_config( + expert.scheduler.config + ) + except Exception: + expert = DiffusionPipeline.from_pretrained( + exp["source_model"], torch_dtype=self.torch_dtype, **kwargs + ) + expert.scheduler = DDPMScheduler.from_config( + expert.scheduler.config + ) + if exp.get("loras", None): + for j, lora in enumerate(exp["loras"]): + if lora.get("positive_prompt", None): + positive[-1] += " " + lora["positive_prompt"] + if lora.get("negative_prompt", None): + negative[-1] += " " + lora["negative_prompt"] + if lora["source_model"].startswith( + "https://civitai.com/api/download/models/" + ): + try: + os.makedirs(f"expert_{i}/lora_{i}", exist_ok=True) + if not os.path.isfile( + f"expert_{i}/lora_{i}/pytorch_lora_weights.safetensors" + ): + os.system( + f"wget {lora['source_model']} -O " + + f"expert_{i}/lora_{j}/pytorch_lora_weights.safetensors" + + " --content-disposition" + ) + lora["source_model"] = f"expert_{j}/lora_{j}" + expert.load_lora_weights(lora["source_model"]) + if len(exp["loras"]) == 1: + expert.fuse_lora() + except Exception as e: + print( + f"Expert{i} LoRA {j} {lora['source_model']} failed to load" + ) + print("Error:", e) + else: + expert.load_lora_weights(lora["source_model"]) + if len(exp["loras"]) == 1: + expert.fuse_lora() + experts.append(expert) + else: + experts = [deepcopy(self.pipe) for _ in range(self.num_experts)] + if self.config.get("experts", None): + if self.config.get("loras", None): + for i, lora in enumerate(self.config["loras"]): + if lora["source_model"].startswith( + "https://civitai.com/api/download/models/" + ): + try: + os.makedirs(f"lora_{i}", exist_ok=True) + if not os.path.isfile( + f"lora_{i}/pytorch_lora_weights.safetensors" + ): + os.system( + f"wget {lora['source_model']} -O " + + f"lora_{i}/pytorch_lora_weights.safetensors" + + " --content-disposition" + ) + lora["source_model"] = f"lora_{i}" + self.pipe.load_lora_weights(lora["source_model"]) + if len(self.config["loras"]) == 1: + self.pipe.fuse_lora() + except Exception as e: + print(f"LoRA {i} {lora['source_model']} failed to load") + print("Error:", e) + else: + self.pipe.load_lora_weights(lora["source_model"]) + if len(self.config["loras"]) == 1: + self.pipe.fuse_lora() + else: + if self.config.get("loras", None): + j = [] + n_loras = len(self.config["loras"]) + i = 0 + positive = [""] * len(experts) + negative = [""] * len(experts) + while n_loras: + n = ceil(n_loras / len(experts)) + j += [i] * n + n_loras -= n + i += 1 + for i, lora in enumerate(self.config["loras"]): + positive[j[i]] += lora["positive_prompt"] + " " + negative[j[i]] += lora["negative_prompt"] + " " + if lora["source_model"].startswith( + "https://civitai.com/api/download/models/" + ): + try: + os.makedirs(f"lora_{i}", exist_ok=True) + if not os.path.isfile( + f"lora_{i}/pytorch_lora_weights.safetensors" + ): + os.system( + f"wget {lora['source_model']} -O " + + f"lora_{i}/pytorch_lora_weights.safetensors" + + " --content-disposition" + ) + lora["source_model"] = f"lora_{i}" + experts[j[i]].load_lora_weights(lora["source_model"]) + experts[j[i]].fuse_lora() + except Exception: + print(f"LoRA {i} {lora['source_model']} failed to load") + else: + experts[j[i]].load_lora_weights(lora["source_model"]) + experts[j[i]].fuse_lora() + + # Replace FF and Attention Layers with Sparse MoE Layers + for i in range(self.down_idx_start, self.down_idx_end): + for j in range(len(self.pipe.unet.down_blocks[i].attentions)): + for k in range( + len(self.pipe.unet.down_blocks[i].attentions[j].transformer_blocks) + ): + if not moe_layers == "attn": + config = { + "hidden_size": next( + self.pipe.unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .ff.parameters() + ).size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": len(experts), + } + # FF Layers + layers = [] + for l in range(len(experts)): + layers.append( + deepcopy( + experts[l] + .unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .ff + ) + ) + self.pipe.unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].ff = SparseMoeBlock(config, layers) + if not moe_layers == "ff": + ## Attns + config = { + "hidden_size": self.pipe.unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_q.weight.size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": self.num_experts, + } + layers = [] + for l in range(len(experts)): + layers.append( + deepcopy( + experts[l] + .unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_q + ) + ) + self.pipe.unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_q = SparseMoeBlock(config, layers) + + layers = [] + for l in range(len(experts)): + layers.append( + deepcopy( + experts[l] + .unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_k + ) + ) + self.pipe.unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_k = SparseMoeBlock(config, layers) + + layers = [] + for l in range(len(experts)): + layers.append( + deepcopy( + experts[l] + .unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_v + ) + ) + self.pipe.unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_v = SparseMoeBlock(config, layers) + + config = { + "hidden_size": self.pipe.unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_q.weight.size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": len(experts), + } + + layers = [] + for l in range(len(experts)): + layers.append( + deepcopy( + experts[l] + .unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_q + ) + ) + self.pipe.unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_q = SparseMoeBlock(config, layers) + + config = { + "hidden_size": self.pipe.unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_k.weight.size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": len(experts), + "out_dim": self.pipe.unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_k.weight.size()[0], + } + layers = [] + for l in range(len(experts)): + layers.append( + deepcopy( + experts[l] + .unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_k + ) + ) + self.pipe.unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_k = SparseMoeBlock(config, layers) + + config = { + "hidden_size": self.pipe.unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_v.weight.size()[-1], + "out_dim": self.pipe.unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_v.weight.size()[0], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": len(experts), + } + layers = [] + for l in range(len(experts)): + layers.append( + deepcopy( + experts[l] + .unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_v + ) + ) + self.pipe.unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_v = SparseMoeBlock(config, layers) + + for i in range(self.up_idx_start, self.up_idx_end): + for j in range(len(self.pipe.unet.up_blocks[i].attentions)): + for k in range( + len(self.pipe.unet.up_blocks[i].attentions[j].transformer_blocks) + ): + if not moe_layers == "attn": + config = { + "hidden_size": next( + self.pipe.unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .ff.parameters() + ).size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": len(experts), + } + # FF Layers + layers = [] + for l in range(len(experts)): + layers.append( + deepcopy( + experts[l] + .unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .ff + ) + ) + self.pipe.unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].ff = SparseMoeBlock(config, layers) + + if not moe_layers == "ff": + # Attns + config = { + "hidden_size": self.pipe.unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_q.weight.size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": len(experts), + } + + layers = [] + for l in range(len(experts)): + layers.append( + deepcopy( + experts[l] + .unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_q + ) + ) + + self.pipe.unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_q = SparseMoeBlock(config, layers) + + config = { + "hidden_size": self.pipe.unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_k.weight.size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": len(experts), + } + layers = [] + + for l in range(len(experts)): + layers.append( + deepcopy( + experts[l] + .unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_k + ) + ) + + self.pipe.unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_k = SparseMoeBlock(config, layers) + + config = { + "hidden_size": self.pipe.unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_v.weight.size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": len(experts), + } + layers = [] + + for l in range(len(experts)): + layers.append( + deepcopy( + experts[l] + .unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_v + ) + ) + + self.pipe.unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_v = SparseMoeBlock(config, layers) + + config = { + "hidden_size": self.pipe.unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_q.weight.size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": len(experts), + } + layers = [] + + for l in range(len(experts)): + layers.append( + deepcopy( + experts[l] + .unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_q + ) + ) + + self.pipe.unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_q = SparseMoeBlock(config, layers) + + config = { + "hidden_size": self.pipe.unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_k.weight.size()[-1], + "out_dim": self.pipe.unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_k.weight.size()[0], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": len(experts), + } + + layers = [] + + for l in range(len(experts)): + layers.append( + deepcopy( + experts[l] + .unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_k + ) + ) + + self.pipe.unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_k = SparseMoeBlock(config, layers) + + config = { + "hidden_size": self.pipe.unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_v.weight.size()[-1], + "out_dim": self.pipe.unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_v.weight.size()[0], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": len(experts), + } + layers = [] + + for l in range(len(experts)): + layers.append( + deepcopy( + experts[l] + .unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_v + ) + ) + + self.pipe.unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_v = SparseMoeBlock(config, layers) + + # Routing Weight Initialization + if self.config.get("init", "hidden") == "hidden": + gate_params = self.get_gate_params(experts, positive, negative) + for i in range(self.down_idx_start, self.down_idx_end): + for j in range(len(self.pipe.unet.down_blocks[i].attentions)): + for k in range( + len( + self.pipe.unet.down_blocks[i] + .attentions[j] + .transformer_blocks + ) + ): + # FF Layers + if not moe_layers == "attn": + self.pipe.unet.down_blocks[i].attentions[ + j + ].transformer_blocks[k].ff.gate.weight = nn.Parameter( + gate_params[f"d{i}a{j}t{k}"] + ) + + # Attns + if not moe_layers == "ff": + self.pipe.unet.down_blocks[i].attentions[ + j + ].transformer_blocks[ + k + ].attn1.to_q.gate.weight = nn.Parameter( + gate_params[f"sattnqd{i}a{j}t{k}"] + ) + + self.pipe.unet.down_blocks[i].attentions[ + j + ].transformer_blocks[ + k + ].attn1.to_k.gate.weight = nn.Parameter( + gate_params[f"sattnkd{i}a{j}t{k}"] + ) + + self.pipe.unet.down_blocks[i].attentions[ + j + ].transformer_blocks[ + k + ].attn1.to_v.gate.weight = nn.Parameter( + gate_params[f"sattnvd{i}a{j}t{k}"] + ) + + self.pipe.unet.down_blocks[i].attentions[ + j + ].transformer_blocks[ + k + ].attn2.to_q.gate.weight = nn.Parameter( + gate_params[f"cattnqd{i}a{j}t{k}"] + ) + + self.pipe.unet.down_blocks[i].attentions[ + j + ].transformer_blocks[ + k + ].attn2.to_k.gate.weight = nn.Parameter( + gate_params[f"cattnkd{i}a{j}t{k}"] + ) + + self.pipe.unet.down_blocks[i].attentions[ + j + ].transformer_blocks[ + k + ].attn2.to_v.gate.weight = nn.Parameter( + gate_params[f"cattnvd{i}a{j}t{k}"] + ) + + for i in range(self.up_idx_start, self.up_idx_end): + for j in range(len(self.pipe.unet.up_blocks[i].attentions)): + for k in range( + len( + self.pipe.unet.up_blocks[i].attentions[j].transformer_blocks + ) + ): + # FF Layers + if not moe_layers == "attn": + self.pipe.unet.up_blocks[i].attentions[ + j + ].transformer_blocks[k].ff.gate.weight = nn.Parameter( + gate_params[f"u{i}a{j}t{k}"] + ) + if not moe_layers == "ff": + self.pipe.unet.up_blocks[i].attentions[ + j + ].transformer_blocks[ + k + ].attn1.to_q.gate.weight = nn.Parameter( + gate_params[f"sattnqu{i}a{j}t{k}"] + ) + + self.pipe.unet.up_blocks[i].attentions[ + j + ].transformer_blocks[ + k + ].attn1.to_k.gate.weight = nn.Parameter( + gate_params[f"sattnku{i}a{j}t{k}"] + ) + + self.pipe.unet.up_blocks[i].attentions[ + j + ].transformer_blocks[ + k + ].attn1.to_v.gate.weight = nn.Parameter( + gate_params[f"sattnvu{i}a{j}t{k}"] + ) + + self.pipe.unet.up_blocks[i].attentions[ + j + ].transformer_blocks[ + k + ].attn2.to_q.gate.weight = nn.Parameter( + gate_params[f"cattnqu{i}a{j}t{k}"] + ) + + self.pipe.unet.up_blocks[i].attentions[ + j + ].transformer_blocks[ + k + ].attn2.to_k.gate.weight = nn.Parameter( + gate_params[f"cattnku{i}a{j}t{k}"] + ) + + self.pipe.unet.up_blocks[i].attentions[ + j + ].transformer_blocks[ + k + ].attn2.to_v.gate.weight = nn.Parameter( + gate_params[f"cattnvu{i}a{j}t{k}"] + ) + self.config["num_experts"] = len(experts) + remove_all_forward_hooks(self.pipe.unet) + try: + del experts + del expert + except Exception: + pass + # Move Model to Device + self.pipe.to(self.device) + self.pipe.unet.to( + device=self.device, + dtype=self.torch_dtype, + memory_format=torch.channels_last, + ) + gc.collect() + torch.cuda.empty_cache() + + def __call__(self, *args: Any, **kwds: Any) -> Any: + """ + Inference the SegMoEPipeline. + + Calls diffusers.DiffusionPipeline forward with the keyword arguments. See https://github.com/segmind/segmoe#usage for detailed usage. + """ + return self.pipe(*args, **kwds) + + def create_empty(self, path): + with open(f"{path}/unet/config.json", encoding='utf8') as f: + config = json.load(f) + self.config = config["segmoe_config"] + unet = UNet2DConditionModel.from_config(config) + num_experts_per_tok = self.config["num_experts_per_tok"] + num_experts = self.config["num_experts"] + moe_layers = self.config["moe_layers"] + self.up_idx_start = self.config["up_idx_start"] + self.up_idx_end = self.config["up_idx_end"] + self.down_idx_start = self.config["down_idx_start"] + self.down_idx_end = self.config["down_idx_end"] + for i in range(self.down_idx_start, self.down_idx_end): + for j in range(len(unet.down_blocks[i].attentions)): + for k in range( + len(unet.down_blocks[i].attentions[j].transformer_blocks) + ): + if not moe_layers == "attn": + config = { + "hidden_size": next( + unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .ff.parameters() + ).size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": num_experts, + } + # FF Layers + layers = [ + unet.down_blocks[i].attentions[j].transformer_blocks[k].ff + ] * num_experts + unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].ff = SparseMoeBlock(config, layers) + if not moe_layers == "ff": + ## Attns + config = { + "hidden_size": unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_q.weight.size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": num_experts, + } + layers = [ + unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_q + ] * num_experts + unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_q = SparseMoeBlock(config, layers) + + layers = [ + unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_k + ] * num_experts + unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_k = SparseMoeBlock(config, layers) + + layers = [ + unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_v + ] * num_experts + unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_v = SparseMoeBlock(config, layers) + + config = { + "hidden_size": unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_q.weight.size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": num_experts, + } + + layers = [ + unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_q + ] * num_experts + unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_q = SparseMoeBlock(config, layers) + + config = { + "hidden_size": unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_k.weight.size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": num_experts, + "out_dim": unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_k.weight.size()[0], + } + layers = [ + unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_k + ] * num_experts + unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_k = SparseMoeBlock(config, layers) + + config = { + "hidden_size": unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_v.weight.size()[-1], + "out_dim": unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_v.weight.size()[0], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": num_experts, + } + layers = [ + unet.down_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_v + ] * num_experts + unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_v = SparseMoeBlock(config, layers) + for i in range(self.up_idx_start, self.up_idx_end): + for j in range(len(unet.up_blocks[i].attentions)): + for k in range(len(unet.up_blocks[i].attentions[j].transformer_blocks)): + if not moe_layers == "attn": + config = { + "hidden_size": next( + unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .ff.parameters() + ).size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": num_experts, + } + # FF Layers + layers = [ + unet.up_blocks[i].attentions[j].transformer_blocks[k].ff + ] * num_experts + unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].ff = SparseMoeBlock(config, layers) + + if not moe_layers == "ff": + # Attns + config = { + "hidden_size": unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_q.weight.size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": num_experts, + } + + layers = [ + unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_q + ] * num_experts + + unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_q = SparseMoeBlock(config, layers) + + config = { + "hidden_size": unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_k.weight.size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": num_experts, + } + layers = [ + unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_k + ] * num_experts + + unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_k = SparseMoeBlock(config, layers) + + config = { + "hidden_size": unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_v.weight.size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": num_experts, + } + layers = [ + unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn1.to_v + ] * num_experts + + unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_v = SparseMoeBlock(config, layers) + + config = { + "hidden_size": unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_q.weight.size()[-1], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": num_experts, + } + layers = [ + unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_q + ] * num_experts + + unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_q = SparseMoeBlock(config, layers) + + config = { + "hidden_size": unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_k.weight.size()[-1], + "out_dim": unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_k.weight.size()[0], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": num_experts, + } + + layers = [ + unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_k + ] * num_experts + + unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_k = SparseMoeBlock(config, layers) + + config = { + "hidden_size": unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_v.weight.size()[-1], + "out_dim": unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_v.weight.size()[0], + "num_experts_per_tok": num_experts_per_tok, + "num_local_experts": num_experts, + } + layers = [ + unet.up_blocks[i] + .attentions[j] + .transformer_blocks[k] + .attn2.to_v + ] * num_experts + + unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_v = SparseMoeBlock(config, layers) + return unet + + def save_pretrained(self, path): + """ + Save SegMoEPipeline to Disk. + + Usage: + pipeline.save_pretrained(path) + + Parameters: + path: Path to Directory to save the model in. + """ + for param in self.pipe.unet.parameters(): + param.data = param.data.contiguous() + self.pipe.unet.config["segmoe_config"] = self.config + self.pipe.save_pretrained(path) + safetensors.torch.save_file( + self.pipe.unet.state_dict(), + f"{path}/unet/diffusion_pytorch_model.safetensors", + ) + + def cast_hook(self, pipe, dicts): + for i in range(self.down_idx_start, self.down_idx_end): + for j in range(len(pipe.unet.down_blocks[i].attentions)): + for k in range( + len(pipe.unet.down_blocks[i].attentions[j].transformer_blocks) + ): + pipe.unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].ff.register_forward_hook(getActivation(dicts, f"d{i}a{j}t{k}")) + + ## Down Self Attns + pipe.unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_q.register_forward_hook( + getActivation(dicts, f"sattnqd{i}a{j}t{k}") + ) + pipe.unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_k.register_forward_hook( + getActivation(dicts, f"sattnkd{i}a{j}t{k}") + ) + pipe.unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_v.register_forward_hook( + getActivation(dicts, f"sattnvd{i}a{j}t{k}") + ) + + ## Down Cross Attns + + pipe.unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_q.register_forward_hook( + getActivation(dicts, f"cattnqd{i}a{j}t{k}") + ) + pipe.unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_k.register_forward_hook( + getActivation(dicts, f"cattnkd{i}a{j}t{k}") + ) + pipe.unet.down_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_v.register_forward_hook( + getActivation(dicts, f"cattnvd{i}a{j}t{k}") + ) + + for i in range(self.up_idx_start, self.up_idx_end): + for j in range(len(pipe.unet.up_blocks[i].attentions)): + for k in range( + len(pipe.unet.up_blocks[i].attentions[j].transformer_blocks) + ): + pipe.unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].ff.register_forward_hook(getActivation(dicts, f"u{i}a{j}t{k}")) + ## Up Self Attns + pipe.unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_q.register_forward_hook( + getActivation(dicts, f"sattnqu{i}a{j}t{k}") + ) + pipe.unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_k.register_forward_hook( + getActivation(dicts, f"sattnku{i}a{j}t{k}") + ) + pipe.unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn1.to_v.register_forward_hook( + getActivation(dicts, f"sattnvu{i}a{j}t{k}") + ) + + ## Up Cross Attns + pipe.unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_q.register_forward_hook( + getActivation(dicts, f"cattnqu{i}a{j}t{k}") + ) + pipe.unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_k.register_forward_hook( + getActivation(dicts, f"cattnku{i}a{j}t{k}") + ) + pipe.unet.up_blocks[i].attentions[j].transformer_blocks[ + k + ].attn2.to_v.register_forward_hook( + getActivation(dicts, f"cattnvu{i}a{j}t{k}") + ) + + @torch.no_grad + def get_hidden_states(self, model, positive, negative, average: bool = True): + intermediate = {} + self.cast_hook(model, intermediate) + with torch.no_grad(): + _ = model(positive, negative_prompt=negative, num_inference_steps=25) + hidden = {} + for key in intermediate: + hidden_states = intermediate[key][0][-1] + if average: + # use average over sequence + hidden_states = hidden_states.sum(dim=0) / hidden_states.shape[0] + else: + # take last value + hidden_states = hidden_states[:-1] + hidden[key] = hidden_states.to(self.device) + del intermediate + gc.collect() + torch.cuda.empty_cache() + return hidden + + @torch.no_grad + def get_gate_params( + self, + experts, + positive, + negative, + ): + gate_vects = {} + for i, expert in enumerate(tqdm.tqdm(experts, desc="Expert Prompts")): + expert.to(self.device) + expert.unet.to( + device=self.device, + dtype=self.torch_dtype, + memory_format=torch.channels_last, + ) + hidden_states = self.get_hidden_states(expert, positive[i], negative[i]) + del expert + gc.collect() + torch.cuda.empty_cache() + for h in hidden_states: + if i == 0: + gate_vects[h] = [] + hidden_states[h] /= ( + hidden_states[h].norm(p=2, dim=-1, keepdim=True).clamp(min=1e-8) + ) + gate_vects[h].append(hidden_states[h]) + for h in hidden_states: + gate_vects[h] = torch.stack( + gate_vects[h], dim=0 + ) # (num_expert, num_layer, hidden_size) + gate_vects[h].permute(1, 0) + + return gate_vects diff --git a/modules/shared_items.py b/modules/shared_items.py index a785aeab2..74c4de8ff 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -54,7 +54,8 @@ def get_pipelines(): 'ONNX Stable Diffusion XL': getattr(diffusers, 'OnnxStableDiffusionXLPipeline', None), 'ONNX Stable Diffusion XL Img2Img': getattr(diffusers, 'OnnxStableDiffusionXLImg2ImgPipeline', None), 'Custom Diffusers Pipeline': getattr(diffusers, 'DiffusionPipeline', None), - 'InstaFlow': getattr(diffusers, 'StableDiffusionPipeline', None) # dynamically redefined and loaded in sd_models.load_diffuser + 'InstaFlow': getattr(diffusers, 'StableDiffusionPipeline', None), # dynamically redefined and loaded in sd_models.load_diffuser + 'SegMoE': getattr(diffusers, 'StableDiffusionPipeline', None), # dynamically redefined and loaded in sd_models.load_diffuser # Segmind SSD-1B, Segmind Tiny } diff --git a/pyproject.toml b/pyproject.toml index a1491bd98..dc5c9c4b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ ignore = [ "C408", # Rewrite as a literal "E402", # Module level import not at top of file "E721", # Do not compare types, use `isinstance()` + "E741", # Do not use variables named `l`, `O`, or `I` "EXE001", # Shebang present "F401", # Imported but unused "ISC003", # Implicit string concatenation