From db3e03cb89cb35f06ffc64cdea36492e79fd8aa3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 16 Oct 2024 12:10:43 -0400 Subject: [PATCH] add meissonic (unstable) Signed-off-by: Vladimir Mandic --- .pylintrc | 1 + .ruff.toml | 1 + CHANGELOG.md | 9 +- html/reference.json | 6 + models/Reference/MeissonFlow--Meissonic.jpg | Bin 0 -> 59760 bytes modules/meissonic/__init__.py | 0 modules/meissonic/pipeline.py | 373 ++++++ modules/meissonic/pipeline_img2img.py | 353 ++++++ modules/meissonic/pipeline_inpaint.py | 374 ++++++ modules/meissonic/scheduler.py | 175 +++ modules/meissonic/test.py | 33 + modules/meissonic/transformer.py | 1214 +++++++++++++++++++ modules/model_meissonic.py | 37 + modules/sd_models.py | 247 ++-- wiki | 2 +- 15 files changed, 2684 insertions(+), 141 deletions(-) create mode 100755 models/Reference/MeissonFlow--Meissonic.jpg create mode 100644 modules/meissonic/__init__.py create mode 100644 modules/meissonic/pipeline.py create mode 100644 modules/meissonic/pipeline_img2img.py create mode 100644 modules/meissonic/pipeline_inpaint.py create mode 100644 modules/meissonic/scheduler.py create mode 100644 modules/meissonic/test.py create mode 100644 modules/meissonic/transformer.py create mode 100644 modules/model_meissonic.py diff --git a/.pylintrc b/.pylintrc index 9e01ea996..c7d03d0c3 100644 --- a/.pylintrc +++ b/.pylintrc @@ -29,6 +29,7 @@ ignore-paths=/usr/lib/.*$, modules/unipc, modules/vdm, modules/xadapter, + modules/meissonic, repositories, extensions-builtin/sd-webui-agent-scheduler, extensions-builtin/sd-extension-chainner/nodes, diff --git a/.ruff.toml b/.ruff.toml index 93f4bff2b..3bc6de045 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -24,6 +24,7 @@ exclude = [ "modules/unipc", "modules/vdm", "modules/xadapter", + "modules/meissonic", "repositories", "extensions-builtin/sd-extension-chainner/nodes", "extensions-builtin/sd-webui-agent-scheduler", diff --git a/CHANGELOG.md b/CHANGELOG.md index 982fde6ba..dd8f0b1c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,14 @@ - Built-in **model analyzer** See all details of your currently loaded model, including components, parameter count, layer count, etc. - New fine-tuned [CLiP-ViT-L]((https://huggingface.co/zer0int/CLIP-GmP-ViT-L-14)) 1st stage **text-encoders** used by SD15, SDXL, Flux.1, etc. brings additional details to your images +- New models: + - [CogView 3 Plus](https://huggingface.co/THUDM/CogView3-Plus-3B) + - [Meissonic](https://github.com/viiika/Meissonic) - Additional integration: [Ctrl+X](https://github.com/genforce/ctrl-x) which allows for control of **structure and appearance** without the need for extra models, [APG: Adaptive Projected Guidance](https://arxiv.org/pdf/2410.02416) for optimal **guidance** control, [LinFusion](https://github.com/Huage001/LinFusion) for on-the-fly distillation of any sd15/sdxl model - Several of [Flux.1](https://huggingface.co/black-forest-labs/FLUX.1-dev) optimizations and new quantization types -- Support for [CogView 3 Plus](https://huggingface.co/THUDM/CogView3-Plus-3B) - Auto-detection of best available **device/dtype** settings for your platform and GPU reduces neeed for manual configuration - Full rewrite of **sampler options**, not far more streamlined with tons of new options to tweak scheduler behavior - Improved **LoRA** detection and handling for all supported models @@ -151,6 +153,11 @@ And there are also other goodies like multiple *XYZ grid* improvements, addition - precision: bf16 or fp32 fp16 is not supported due to internal model overflows +- [Meissonic](https://github.com/viiika/Meissonic) + - Select from *networks -> models -> reference* + - Experimental as upstream implemenation code is unstable + - Must set scheduler:default, generator:unset + - [SageAttention](https://github.com/thu-ml/SageAttention) - new 8-bit attention implementation on top of SDP that can provide acceleration for some models, thanks @Disty0 - enable in *settings -> compute settings -> sdp options -> sage attention* diff --git a/html/reference.json b/html/reference.json index 8779ef603..2cf490c00 100644 --- a/html/reference.json +++ b/html/reference.json @@ -314,6 +314,12 @@ "preview": "THUDM--CogView3-Plus-3B.jpg", "skip": true }, + "Meissonic": { + "path": "MeissonFlow/Meissonic", + "desc": "Meissonic is a non-autoregressive mask image modeling text-to-image synthesis model that can generate high-resolution images. It is designed to run on consumer graphics cards.", + "preview": "MeissonFlow--Meissonic.jpg", + "skip": true + }, "aMUSEd 256": { "path": "huggingface/amused/amused-256", diff --git a/models/Reference/MeissonFlow--Meissonic.jpg b/models/Reference/MeissonFlow--Meissonic.jpg new file mode 100755 index 0000000000000000000000000000000000000000..ee9aea0ad269afc4ee00c5b31eab3d9d718999bc GIT binary patch literal 59760 zcmbUIbyOTp*gc94l0YDVKyVEXg9LX8kQv|9>UnBjW?xnTAEm)kU;qLF0>I$)19({jSb*Ft%>V#7 zIRFp<0H6Yp5r_b9Ue6F-voHej|2YRE&;yYErylXOkqz)#0O%8NBBIe)B&^ z2SZCcCl)(Xr_Wq0tSoFCuOlr0!~tk1D5xlJ(NIxQ(b3S*F$g|leE5Jtii?LuKt)bN zO-W8kLCeVVg_ePnfr64%l#P>5Kv-CqhUJ@-xS%ADkg&ji7ePQnN5}YpLGtk)Jy{}G#mN+nkH4_A5Yl$yf` z;{Of}51)XLh=!Jqo&m_o#m&RZ$1na31eTDLl2%bwQ`gYc(l$0RH8Z!cw1PT0ySTc! zdj$Lp3T)iSg*Jck&%&*QU1e)fav;~ zk+6{8P_Vtl7F9wqbo@lg{sR?9EHCzUSA#(7C;2Bd6~-M#Ra8^8&>B*MJ)5rU%euM>M*ZsI#=RZ*5k9K zFt5j|QrP@Bc7MJW0S>pQaGk5p+H~VFs9y_bKr1_)7C_R)}-Iq0NEHG>X3+b z7=xErEG)5g_&{+yFXo-##YmliMox0ahr>Vh1e_q=@ z+8gQu+c9`|JDiyXL)UH(jo_4K5nw7!4{o*|vfAXLTPjU<5Xa)eJU^@HLTU)r?0cVD zedRo~3b_5y{vpKEU0QfpK>_4Qrv>B0t-qK;>!rpbI9)yNMfN1Onr}Fjo#9A5u65jz zc4gz`bs|jCU6n!pm+RYkdGA~yWq8n3s+5!oXW^pgGpjvu%%ZFd71SyenHxNpk2^t? zU&qhs@iKW9E4EA~st;|Evab*~E3r0nq2zo}b#NQTP~hVlQ)c0u_q-2F$Q!P#lKI~D zAocOzhre}!f1yk!2y})c=UlBDjJ7^606Bf&^yGZmJ8Df`cwoW~)n^mq*|DEvb7dZ_ z(wz-`7SHO{^t*VkypgH31dBzh`#J$6>!4MIRPbh^|n|A(z5A+=6RTg$?t?|qFv z;Z;~eXq!hLxrF>r8~v#Me&dkB&`fzBxgzrKwegh;doJpAH~fZS=z;c2aFn=)lY6xS zjl-cx<)851X-8GTUh#0N9}Wa8QkN)?&$Y&yOP+aR=U+~!{gOxlyi1Q=9`5S2kocd= zBlnLA0)!xt9<%&sW@)@XKi7%y3edlPc&8lrOB_*kjUKo6r?UYKn{J4Nx|kwkOvrw3 zRQQ|TZ_XwXpg_pSL=(DNJVXRit#=^5fPpdr`ro*ZO*PeijCaoFrT~re{qBAMY-=hr z0YPJ+Oy{*#u&H)y^o=;F7QGHHJo5R&k|vaXQvYp=^>pyhk3_eAOh+NsSHJr2BF>h9C5s_}7*oC{n^Oozx< z#kM{cohko>=xu*lQi4TQ42)^x(s9GK`4bmJq`d}RKpOIvM-m)Z2->kBUHArKs>-ox zEPq22R6_DnP{UMtG)A>EBwqkLHxjgggV%^+iL|LT%5Q?U5G&(slBq+`RIWZwr_KdL zYxB{P?EWk;%hWa%><+#t7+|l6L`(!Z$38m5J|8H<+jkq3Q*oc*ogW^g4i{A)I!2X& z%Bcl|$OI*5Mo|WolvyBQGh{d6gme0B#81k&xvL;$o7W;NCXD5;8N+Pu#uvA!{^9P` zE=vaJR+6(%*)DIT(j?PiEwNeH{W5U^V@EZf;giI+bz$NfJ|XuCGzuC~n4cKCon{|? zRVE;59JCh(1mtwT0Ja;StbPB^bH@v13~fCXu1~M6nV_9*=*RlL3v@dYj$u*YmjI_1+DZI$W3+D~*u701{m*8j+D=$D;K+79TK#6YHA? z`r{-!vNX2;6vzq4!Y5$!jEqiE!sV(aRabP4H*}|^WYlMrZg#ybr_$$6QG&L*FMt*> z$K^1PFQLv@T5fg*+Q_lhyt?tGm;82D(3|bMEoMlITDkOyJx@~RH`c#DF=dlTe5k#F zlf*`eNr#u>?wgT?eGuQB?4jtvfO~c4(u1_53g+@@sv_;&(ENmr!I)w`Y7CD{f=VId zPf7Ex_LnsRrauxaFL)~L_q+35f8(r{B28Gn07R1aKuby0Rdsgd*-=P%R)ok$ zJipW5cI+o;X!8&(o4De+JTjZ_PzaYBTjo{MmZJ6?Oa#$U6|uM@74OCCsvNMDn*a}A z0H&VLoTbX!Q_>?RJO%2G<4SF-9_r~Z_Ys1$bq24sl=)AI@c|jGBu*m}f-_l`+WjNU zfw|h!J_Da)pGSxV*!|H-FfWE2#1GJlsHbLpZ;huD`7x3ONwbY}>MkzJ?$$IrE8^V0 z#z(|J^3HUlyQ@XkJ|{n8GnOSs9*wWtt}|#H`?0d4UOt<(5+xMSQ;+);rNIb!@a@G% zf7ab3wU1x%fd3lS^j_}VzX0yb&=!=F46xa@2k*|${H0z1LiZ0d!!!N*JY3}ug&uIn zzTND*w6ZlAH{L~lnnWaqG>O5J17C%Yp4fd4+ftQQiYe*FysD=YY3sgmyheO=iSwo< z(@3J{nDylB*@tsjBU%qjx|#qzFX3a2Dzm{9*MSC7xiHkC)p_R3QhU0-p+)bemYGvJhgMXcT` zGpMKW|5wU7`4uXcb2{OuGN`|I3Q}WKNhokk*daEg&GzEfX1l@tG+PXlwf-&hc7!Yu zd2k_0bFxP~hxr39#)1=423ncPPCp+V;d-u4%yhvT1zlRWHwAWc#Hz|-hjqbI!3?y2 zh2isKDSzy2R(7Cx+U%IJJ*1JZG}`;mn{$IWDUN|JvgV_Xswo-FlhwE>Yoc`XnFimt z)D2NTZ{o(RVk}EoP@9q5}JAGnD$v627*5EI{O zVyZ}*J;kxl^)$perZxPIC93>|*ht*mn1g`JAjS~#%HwmOzX7Ilb$%2;pfVf*0hdA> z5Ns={?ngDx@LZHA0@~(yuOf?{s(4e+3!|ybmZ?+D?c6PW2j#5nA_@+{@*JR{h0!S@=u-0PK)K)#{N;bGuDjfmv1g1Ln|==lz>8?#K|- zOFrD#r=)62tb7l>1(_d>YYOr&0P($hjX(v)OrkTZ0@!Bq!Q|C43zr2Sb>_X^3m`W1 z!;h*&+oOaov^c_vVt}2EFTC?dR(mZoa`)_ASpUCS{@+^)Z4-}}()ZR`Too18mad8m zv()seY@&>TBFj(KnBr$F0mkc&`E{ur`|-9BgNh%+Bl}SVSulKw?heh!1 zmK%pDVUpSyF`eM(`GyoW1#_fF5e;=$QaGw{WI#?an`y?VGKl zq#a1QHK=kGa-(skuWcW06VuyB+JRO=HMgKuP+Lwx_0tBSkqVcYM#Sa%g5M~crHHC* z_v{z@{9`~9S$1&2_bEHIAj03J{6?)n+yiD7$El8S4>(=GXKTU=`wF8(anM65W&HME zIv>D3FX24~!#zz0R|}peAdv+*_KSD9kIfT;NTw(Jc;m#*>&c}NhMzOKe`H;*S($Ob zyU@hwC+r~fH?x&6+e9f*t*X^V&$lwBXoeb>S>xiP ze=K*lHM-7S-=fgmpYM`i>;GCc)fE;V%z)UXfyd86Qs9WIgXO8@u*DeiR60E>G(^&C zg^Wn0?V)N|smkwPATN5a${&hP#GJaU1ZzH`??|N}jBa>7;^OSw<1}&yR}h@dS8PHO z1xxFHzg`7+r)F)6kB-|?T2ntC9vHT)r!{v3YX2Vwyv|)Ss!vpR`ERuU?T4Y969#j4 zst{IA4TIe|>GDaNeX*)D&Ry$QVpHoN#V2RT8l9TJuVoCT$QYfP4omFmHz#N{Q}3^xpp6k zB&VlA&06x@F2zTFj9FLg9H#9M)IziT^kl)>rqdKYM;dtk{kg|xnooA?Wvl$>Fje3m z&5d)?gVbdXA=FxWT5*$orj7X=QNq7hrPnA=rXL{nDn1R*Sq!_hM@tFa+l`CZFfCxVqx)(s-ztkmI51ZaY4nD2yt3+j!`k z8FbWmmjOq>O!y2r5cDuL$i-l2Yf zj%=Hcj4@|4e@5f{huY!>A^mouiCWp{WR$m>$OdmY5CuwUNuh zHrAkS3@gf7nW_bzJns>+(O75qBV2MeX#{(w#|4ipQ32nR%J%9%WO{clF4+B>`5f$< z7jToy-lMGUCK?9mc*n&v8Zb_kZv5;OI8MNxg-iJ(0Px)pBXc)S@|z_UCy1?qgO%`l zpRW?T+^W$t`b6lK7qV~Y*6zkx=4PM8EwFi>IeHP9Um}tO#3ZP^9;$7HwG3@BrO$s2 zOm;*@$jX2&w=z-b4oPJYv{DQvKe7!Dl+J7GEF75eUaz)SyW8$a2E=Ay_-&e{0wr1I z+lQu()_)o?xa&q>U@Bz~%!kdQ;if~>+l-Tntrx)O|y^<7)W(s7}6ObE!<4 z$7dM70AO09-;y*S3quxLnc>t*8*~pVJYV@pu67FCYx>8tnbc-HUjTm_4^@U3n!bFP*STYicz$ueV%KmH30oR0Uv0-(P`y!W!n=h{bre! z@9TDtu*Vj@2APfuKkAM*nFsdvetSZ95Y&!Xx4J!8RtmM9e7TyfQ3Y|PU_K+Ru4ZuqB<8=0B8UHoefn=6oge*wRZiQ< z4{fP=DZ05w=rdu_e8PpvtdJ<`tPHd$t&RGK0l}~kfq3adOKjuu#z>jY3vA=xN<`6j z78}%0C|02Jn)-27?<&ly(k#4Y&jnAJP{(avir=j7HdT9z%Q#>_HrC=)DQ@-sm&3(N42t-X1GIkBY=!*k6sa#Wx-43?t zsiC-mBw5DiAh-&L)1Y3WX^shvra@S9Y`^-PI`^wCeE#19<6#*weZ!sf1F*Ya1O21T zf8P$(|1+jp1aKo>odJx2VSm)L=RiG1|EnMZID=|7J+DH87d{V5*?Uz3PmqSJsxF`G zc31^mwL(~hE4K`W`>H4dZdKeCpUAH#UI1daB9SWgK}8~$<04P*6;mLy7uO<9IE&Al zD*L4GVCi8&Z@U->CI1lRVk(Qm@`UZl5Ph`Pfgt6YlrB&7O2w%_%`5)j=gX8@k(He0 zx7tYt6jR4onjbr}fWM9dwYgnc5D`kB4a=&AE#~1+389FgkcwnGh11^;g*6@9iDAlY zYyP%hlpAVN(cHsS1NQmV&qZnTATsIH2wgLt?d_SL~6=CV_T{ zPqiyqbQ1TJZ9HT?z81AjFSgpOp|Spf3|&9eMf;n5HG@Cv&aqu@&$Gb&l-_JQC}ZVX z)_{EJN7%8KRQ?{Gc12`i7vwi_4s|j|jYIJ-sERx0lHk*kv6w?D=|NL=y9AZSQ?|ML_+8L-1Ix4IEFo~2FF>s zzjvfFH!cfP(iqrPhchM`%GFFt`js526k#&7v)!8OiS6BbqlA2%$;#l)wbI1DBk_cV z(qK1OEM3vS-$}FstQ{m+w7# zWUH9%F?M>^pZirGHyFZmLA(8qg9HM`$?k5nCN6VxYJPAR4`ng@7V+b+LFdKZbo%uE zHJ`a06fDj2=kT6r+FO{La(I|`#E75Sspk+3 zdADwq9ceZCRnq}xokK$nl+YeV6-$uo@hiDMXY)1MYLg&9&Vb>dHn4&k!f_?rhA3!K(xL>M}f&cQEL3KgyA;WovPu`+QZpUU2xQc zV6t7kNO?*vHM?Z#E%T6}oFmQFzffq@&Zrt6DSQ`F)(eAZiM4{5*W9?fS0{lgW_a7*22jiI4!;XJIh>0>~g0@Kdq3m{NA?));32?`#19(3=S z+`)yjrMQ(b3xnAUl;w}os{XK0F8OJL_C27Q(+(|7fn_6*@1pD^z$Bz?67v_pl(gHv zE{rlqOm|?4ETL*DQQo?z@fZ@Jx!+%aPunR&s;&BJLc`jR>Y+y}OLX^4+0sVV;)I*3 zV&l9ZQYl{C!Gr6I+RebsRE~3hH=G}$^V@+ZA*;a*R)4Ht0A8ngkJgcm?uNR>eZ2%+ zQo3FxM9DnK_&6O(*RhXBP$X5(u~TlO zA1q2ikCLPeUjm=DlY-Y}^o%Hi(COsC?A~m%uf{~$hS}%|Jhb!r&?MlYpHIKIsA$g2 zt8K8{cwm_fSGl zW?P^msLSwykxF^m#_5vT-7PeFZCL8@4N*`~`T(|T-z9Ezp4GL-A|_X=QSpNOTEJEQ zvH$k0n34{`d)^}A-SYCp!uP63?Gb5N)_95C30+de`DaX3=jH4S%Nw;Jb*c*%=(zo= zTNb2Y|M{x@yVRqK7{4B>TI~PR7MI-g@<{h$c%`etD&JkBg4+nrU@Tl- z;Ac{BviXfzio$WAz6y(f=PVZy4<`q$o{D0b>TKM&FsVXLablGUjguZ9q>n>F#qVDU zb^26G7xKoU^8UDcT@us-+*X7;A~%kpPyDQrap-xaD3Xv_IMzB)KFT*M6D04b>Y#;t*%ALH(=F^29XbrUTPz* zsIXtCHBWN&axu)F z#cp%Qimhi$St4JLml+sRH5a9@$)Rz4MWr(S$aQ8kzm0n6+%UfUl{>WOzV6lW>C&?B zz*z}I_l$@8syvqQpb7=03)S7oR!?qlfU#QsO-WJ&y8Hfo1PTA}qPUp1mf&^!lqkN> zN5}PZk4kbVExF0&nP{?e%zyfv49pg*XL|>qx|}&*WN$iFih^pw$7&}Q`}JzT)pWoP z%F^A#flmwds)F8(Jip3is14LYKO&6({MVVUf>>z!SQ=)a-8(}slHvXc4T+i<^48Zn80Z% zj&0&~DhjGZYW8u?1F_va8td zR-I4MS}oULocMFit zZQf7NCY{S+_NR@1gA|};uL+?Fr&MM??ZYg|292f0TVylo%))Um!{{;uLu1MsTlv+b zPUgFhuK(bP23%EoBB?br)R$Z|)?vvjrD;=nEX$agwKTxfEHz1N0q#;65D7Z!PtjH* zqgSdHVa)&?hHM8Zq_ejTRaa%*c~Z^pq>V0^7`qiF=@*mn$_qwI5QG~40v`0aCZuM+UPZuG=E!s@-5xMnFwUNMir?QEJo`hqBV z#@&+k4CCXxZL6tDK5ctLs=BZ_*0#`NtE$X=gR(fyRfNUqWUyC!{8T*iqYKzlOXf%o z0V$D=`d-Z%My1ZyTfJ9PR_^DrdVQif>5yO#EzM*A&39*C7es_0i4*qQ-KA4MKo~h5 zKB)z*vpuB>uLm1_ln{EjP%Ig@U)Oi{s;TSLNUIp(N1aL_WN+31u%IB3@1!}+vy@SX z^+;q7qU_9;QwNvK4KmUoa#(V$|S)@2w3^X?T)$^MoZihWI7D>a+>r;`;SHtPZB3TtJJOAeIy zhopN;ohaASO|wyTQ7ee#1&PDf=DM!CJJ{fMYMHblcr?N<&Xm4xaGAbR8$ zwaCht$M+OVYsktH^bkm_^xXX1ztKi?@hL=pSABsSvpWa#TQ3wQ9w%S@hl9#3*7=Wr=(6M7-x%lOi@po{zxLe zoRweS$~4n;YC4W;p~G=&l8&u@H=LM=LR0p$)df}BLbi*F^ zX-6W}YZlT#V4UbMhspp;d8MtK?I$|Ps-S*4)^tk8JOG^S7FF>~jO%&oC0Lf)x$5~; zMrPQ%Y^Ur|V9R+E?BzX;nC|8}z;jAZD^y`AbxH0*1^AB+bwyll-KZF0pBFY=yct&` z#34RKg(rO$+wg_!@4OZJ0(oqMkc6?6g@(`#r=Zvje^Bl?-0>^NO`jR_r5Y7y8FkYy z25UmguYI7;*doG~MJ$2eX)VreXIU>u|7|T;z0R+T()-BBt{SuBY0bL>I`q5C((Sm# zbXt!#e?gg|5u?))PRBmfz5#^T?;>xmrXq56%uyw76i{aGr^8GdqRyi7oITfd=BkK! zQwLqKq%tbj6?dwTkWNeMn<7*i<75b?9#UqDMFNElA$gVIJ@2pWaY{`tJ|Z@*e&i2Z z&3cMPDauFx_xu$bML5~l6e@3PU$sh;9+sAjr#z4=j{qMni}rSILml$sfv)YkE%X!5$!6n*cZS%qB#O-h6d~SI!0%5 z_IlZ8FJIuM?aA@wkq@c*N?(!%##WfpO%U6Uy_rg@%_8tvvxd2x_ObHV>^}2VSy9yX zy9!d&gka?YJs;Qqf|oh~blnktuH*na4}NI}qc>qcC}7mg-y*@^kG5 z7~%1eZS=ZGn;>fyY@(@lcR}ozx3XMkoyb5teo*v?l%AE>qODN%eWjzu5_qFtue4O= z^+$FfXar`xe{-GGP7i;Yp^lKDyd-&Cc;*8WDiyz$FtibY)}wD{`L(U-k@fwXV~i*HRz%#2Vd4f={3P*V zFh-{(U<+e8yT3wDs!Yxy*<2yXHRd>m4awq?Wctr|b9hy9b%E2066X@`k^a$b9Jt;= ztuFG{ecD`e%T%&>(1hH?cG9|mS2q9&``&r0^&pJP(HRIi->q-Es$6%L&LYEV^0*`5 zuYfVM&I|9q0Nz3~ZiFxE(u50Z?&2(!<7cQ#K0iC}HHF6~DK`EiV&T}1DChTVY}u_R z40wag@hS1LOIwakfAR}@Uy*dxulCM{ym3o z$`Jy0wXXz=tbjvrIq_0U?a1SLlQD~PCYK3`)PIVaZ&3<8Fo&xy6(JeRJFHPL%bcFQtUDr_(5P2;y0w3JK0RsTtd? zWVT|MPtbd8(^1&ES0AF*vHaNNvaIvKO&bzDP9`rXDYWgXUeDFU4y8u_u-;mZ~%*)DK z)RHgBJR+x9(w<@QH`3f2Pp(f+&;HqfZT*S4Dl?i(AeQYh&Hc zz^7J}F{$!?0kp@kcSDAvfN_*MpV&69OGn(F#w$#|saUh`Mq*~;! z^>BN!TiH}5p;yWH-XsPfqy`8~>4Hqd$*z#+)8drJMfid>=ukyo z&D%ga>ZVAQ_+1?|$6X;g6aE1wog)=OfQW-v$;3q4hT=XI2$W1ad6Y|IwB-1CpK4dQ zs6dGx7gZ5}^y^4wo&y9H(%i{9KE!0Vr{#D7Fs(nsEvhr`d$s8Qi++H^H09$Y&O8*B$L07SRKb;-5}-w6%`|tt)3!Xo+dQ z8zp#e+I3glP~;E!|4Hn-_3d=!*2oh*!Z5cBaU1?ZS#iTq3`>MR|_UcJ2Cd_wl&x08W0-X0>*PCGzu3Z&3xRLwNZy&62a% z_+KGIi4hZ{it1#@t`~zR3I4e#n<#FlDPcy;8A?wh`5_l!j%VAn?BC2hBgyV@{IcQwkf5KB>|oW3H#AL*t?Z+z!s*X-s}vz z3!%kj(t)WOiUH}Nxj+aH2D;UmC;AKX*+d3wJVQWbH zy-&ittVUIk@r4=I5m3iZ^CRF&cxpO3h-lz1=@-K!xfY7|Z^tD`c8Uo1WD+s9zh`(+ z<@=oPd%~2D6c|7JnN%ozG(S*fX%Q-&_e#ba#H0LbF!#M>V;@BHxprvagk&#wSQCE4 zvd_1K*26hoNF}oT7lrq-LV3Gg5^TyV)Kfq`F-ugpLEHAHRWBJoez0+PCMGsp*QtzE zv%32o0bSa#ibgd*1*q0Vk!>CRr`H?kK9oqsVQSH5{~O~Soga#a?hWIdjw2_`?g8Y& zJ(7~KwQk?Evg*D64yK2?auW^0NsFjQ!HZg!12YadW#PY%;`q0Bq^p*=IqwzhZ*1R^ z`s7oaT`gn$PE11CY^~Z_$Xsh38$72@qU^}tEXuN{xt>;ZhO7u(9ba; zJmqK`z|gpz(vvm{Zb{+)N%FWrKJ%-J@;#Q|d|%q-IFZB&*!L9&=Z-?~_A$vatw7I-f+c~$CCYqC z6Vpk&*$#~U9E)a&aT=5Fj*g~52Kt6bpRs1nV2qN=2BpTThLalscj_4RttWa!-S1#r z6txqwoAloW&h&!9tx6()pYZN`V7Aw9f2%bj{n9?#QzuhASC4_?j60dj`!LA1`vzZv zvxu%Ll_mLoJyuSjS8+EYT~Vu*bolRN?|>QJ{7rx6TBEjE!$_UJPA!rcAg@uDZEyKE z_lF(9X-Ih;w<>k{0x51m{##PfCPH4RV`aCxtcuSf2C?z>4p79OzdaNh3-aLh{=S*^iVnWJb7u!SP*aq z%^cpR9xKOA_&Fb^kE;3AJk%0Sc-{Nalizkc#oT$#uIpTS#Pyfyk2dSbhlVJjZeT>j zkh`Nz_SKYC*HT3NF1FS;D*~S5nJl4^Ta{f^z5LWn+BcoP3QtM!wkp_Cfb<0)QgpVx zjZ2rbx@o^9>>X!(_#5&g@*P34Zs<)UV5iVb$XksOF56c6Vk?nS2Ugho4pc zrw*K|d`$#R1B4&brEFVZXtn@9@yPA1h^X#y&N@BpDo9HcXn+*2*>BmqTLhNI=Q4Ph z@SRcEWh5j2vN9z4XAefOSw|iLOjC@D;Dt0H?To7ocNkgV3xG3z0%wXOY_xN+2Qk6pF|&P?_Z;ls>iAkK1bdTK(*)_gN)>Pc+pFl7$Y!1Y*xsXT78 zgS0or|Ps5V)?FV6=9Oj-@5GbIh|*>22iA_a4+e0z`NYar5sk z)h*rPwlx7SrV0ow)`1+8jtt%@&VmTa%NdmIMQ`r2 zMVHf>apUM6x81{X@_mzVg#gFhWh2n71qFu|JYiLze*PCgw$=E|Xwdkib!7_md4{1p zb;6bLE|3pvqu##@gW9l3{AuyrayETeEyhqvGp6#UY3HH@Id+IMspBU-o=9=of-)m6Sb&zT~u{yp&0TrhFS(eV$B$!-Ui^!!(fcaJiP8)gQVbdIq}T%Xzq z3gf9mM>x9$r+(OJr4iPDVH!(}Fm9ww`_<RPZAFxQe#0SIP(jhr7-n?O1fG3lHBL-U{YmtBJS-So(0Y`!Oc5Z`H;~BxUIU48+xfF=$wRW#qRfc=AdL#)TRyHn60Lww3){(scOJ}W}3>>HZ$?+ z4u@Pm=y`R~d!f5hBA;Kdt%3nl)DJjI(>00jbg}~Oh?yjq8i8e`6^VuA!8<0P?=qj1 zbehVgar`m)P++!bD%LAF2W1b%d&G*hHHZRE*anb}u&r#Jx}bwTJ}Q!?3-}W-ksyzb zulL~cp`=2oWsxi&cjL5N2&FS7y}O<*F>pp8{RHm?aHS#SJW$GA@OC)UHa6W8QK4~& zQ!s|jgv`WtgsuSEaKZe6WqPYCI_fRMnH)*%i89;D{O^Pvd~2LIk7&*ICrhXnUwxw` zHAwLE8)ho7q({E3K-B9Oc#w$NfE~R;-m=?boK>0ci9`M3_ymveXI$`Z=`7U@%8$D# zA+)W6sfvi3u*VF31%HuMf}Slkc{%n*^}%UJx{N}#d5C}tX+l4$(c+cWapaxsyoMCf zz7=v@qkLib*Ga3%W4UlNtIt;&6uDX|L!lePJnzlY$TGW(5r$@Z|z~AhxH9YbifYhEw#H3E{8|((FBvW6AK52}rU5yq9dOOAl~V z?yjxsBzJW6P$Sho)oE8#5Mtml%wO)Z71wW)0LGh26u)g0I9O}gDz&pKnVpUA85xLT(N@Y~wjEozyKpaccW0$ucqGIJ3nI&_KXz{lu!)~W2+Q0`d^ zJ9L#w6x@i_MG1TaHGPZBW)j{TStd)wIB?{ptD3W7EPe;I_#8_Bsm9Ltp)BuT^XES& zFj0kvgHeMZQM@#p(`@=)lw*Ga2MXnjs?X@K_nBtcI!N z$%)_HWop|hd(LyM=TI0&?G{Kl(60#M$_Og4UD}yj5_>l+>lwV4)>2;1UXFT&!l^5J zR!JsD^32p2$Nj#LBH&iTaAOJhLHR-sUjO66(_p&n4Rsm4S^U$z1lMQk-PTNT64SUil2swzyI~|sY>y~&#tpAdJamFAb2l1C`m-c_>TZY5M)BEKeTpc)+_D=qM%s&;l zSDlpBW-Xxmpfq#qse*U&>DUXv1ARI)7D-~ndzXq%t>iyTym&EED4gh@F5Nq>7@jyJ zu+Tc*QscGg6(IOzjdp*?WP=Xz$SUFlv&d49gY9aa#urU9*m~>&=4(vs;Y&=`*_nTF zcO@2siiqW*)%>KuFO2fxRBwVdZl_{HM(KC+aJT#AAi!T7_!3)>fj4StdT-A}85q!rLT0le{@CGFpHtQALug9Se z^z{)-BKJMy&E!jDaHQ(RpmP;elVS7ZCqvS13Qo z7SFaeQUFB2y}U;~`djiXRVFoI$2^u)bS8}l#MdKLhd#M*h+JxWHcR=+SS|gROW)u7 zSmiGO^E>cyv*$;TfLFg0uP)boL6xvgcC#V|yTaX&G4DTN^PHxZWCm`SsVQ-biz}!) z|1iG(5v|fKPPzI>%Go}e>3*>`85RhKp&24$GoOgw}h6vNr7M1Pt4~t#6J|>JRcOcW0*QBC?wcLS@Eez4^me-zYPtP#@G{Zty;A3`Rsqs zT;Si^!v1Mk$U1Uj(^{qqctcPO6Q#*TgYJ0LOh|=+`u^dLXj`KV_&Xl#>G}-8G9%N@ z`N?Yqfq;(UtIPXo>ZSxM9{YbnHR{DhXIMVnTT}Dt5^h_Os3ggHvyY5Frv=5vn8M%f z4DjATSH^QzC#KlC3+h4494eMvcN$0UDDg%3>H$0?t5J!aky<%zmRg&Oz9Eq<-a?F4 zPB?14&PyYb!J61s_GFI;H=<-$+gMf_*azH^JFF^DGWUI+u-!xvC&luNxT<3*o>tSI z;=j6;8I35@4SZUA7gV-p<=J#ZsYhF_c3$&g}&Jz$ky-M>Y`|5$t$x z6eZgni0Q)|KKkWAXONY=3^#bmAk7y7wK(ok9~q<+}X+S$gi$mZ+(qk!VEKYs*U}b1%?za z{k<;2JC>bvcQQjsqHIZ&K%%-+O*Y0a_jz@LdcMJL%*)dsFfo~h6w0e!3Zms%%rZaU z+^XYJX6@*b?HHSS#A)|25JQd~JfXUm`0Ko~B$>k+dl}Q%g$rEap;LP^Y|EUs#C8NC z{f%wN%YyTpb)8$3^^3y!(Wfv=(QXaaF5Yu3(gD^e_NnhjeaZr8yEdyzjYDq!7XuQp z=>T<+P`d!#bc1F;F{u&ph_J}H37j}SQfVy~^t;Y8?xECGul#4ugW1!Ihvn!eJVs5Q z?2=C&UM~Pq<&QFsW6{q+z^N8@*B5|$+oi3^rT&a@E`MA4W53d0?i>SS>(NBf86pub^~$6mSEwsc5&70Bnr12Gwa32@0ta8| zmDAb`j?QF}zu?2HNvVQ@NGg{~C~U?{iaT}9Vp}wPA#j0AVoA8@wETnE$peHp9?7bg zh)aJ>)&S`wBfCFoWSJvu+pZf_uw7I9$27$tOB3AqtTt(>t;~mZ8-*+)VE(3Uj(l?T z@ETmdwUp+AAEk*W<-$Sb_iG0by9`^gy7!4UGD}m7BKQlS9~lKO&>!S{N@kxNW5h~7 zdQ1JucCL+_dqsAJ+K1AjH4XX#u!$7by5_hM$aLA?W_q+P(3k(+Lq%29ei!PeFkR3s zlVeF&!*6tzavvdDm#T=h;80S-n9zQkJ9o;w^5pzK@iN8RXt%gq?rk>8IMZ-H-@RXD z`Ppol7V{`Eiq_@pNK&$56S$YE^f^+G$1W!7_t!fQ*$I&?xS;p%pk@_4GMwiMx&t|*%55nC)rcv^K5_F3sNQ7W;zkK$^jO}7 zGst+Hi~07lXxH2_OJA3n=^I>$=b!j8#QQ^E{sv-+6!(U%C0V$st@eg#iPBU<`H{Rm z`z9C@mG6G~a$_o^*2+798fnm<{>h|JdovNVx_Ys9qS}R@3%U|@hfGL{KndF`D}V@2 z|9;|)*`u-Yt=CvCh<}^9no6ee?%|S`Xh8YE1YBAACjz4mw~b0J(C`d3Uf1jvQZE6e zHnQX^vq)VXNo(nv%3Dp1rEkUJ>%;hFZi6?}Q^V!{FQ(2qtgR<{^C{X=q)^-o6o(df zFYfMz;ub8p6f5pt0t9z=hZc7U?(Xgu=;ph-zkT+fJh@5ky~#Y2nKS2oKW8x1gJc-u zO(f>Pd?gF%1ofl2A*=oa^W-zFLhbV^?M{`aznUiBq*PFUh7T-TOOvbf=N0F|!$5PR zl+UtwnCwzs0dv3Yv=wF64iPxR$gm7b4*cZ6Y(=<_WVbL$Ep;G>H5mAn3zCPtz4JER z5wXHW^}*)JYBNrpTk)O{g4OY{OtPmJlwUqg*ui_3xV(^Fqk{nqg2R2BQnPp6Lfbej%m+Rh<_Xg}?+lc_u zjW{X(jI92O<3@hwuhB+Q&1SNd3d;DWhwVEVvm_vM@Nmn# zeY!joxDq&wrQk+Yd_X*~RDoR6szYbM>5RLw;)hkq6rfQi<({0`R&~Wsr^H2VCG`B^ zeF?D8%AVRR9+L>}k)D|4k|;hx(e0GQ^|W7&;T`k7DI3>#ER`C)D5dZBsdndo?<uf)NN`=Ct29wSsM$$c=`pq9(8%j(?1%yz89-zr};deNQ}c*Q=F;q9+T zUu+~^pk*lMwU^U)j9s(G{$`@XxIFM&b(z}`#RDVIdL|*TjY@FZO;S2RI{xtsMA^&d zq9j2os$4+%ea2p`x+4BY|KZW}k=Dqdk?_}nNqpQ>dKp&|3pF0!LP2^WW_NV75;jTl za!y;n$5XmPEuRi!2gdv|!pPi>ql53a@EobI0Cc*Gl<#q|C=_8JGq{j59bhZ*|8+2J>vtM6`JID1?@ zrbhN^sMz{B{Jp&1=;8A;5l7dqEG?|cesr;e8w(bTtzD7U#RWPTS7)$2b^|6Dz`kR! zxspzA`QP(}vPPFVlJ6Rw?;EEh=MdigcDiO(5PMuwr9gEA0mVAMQ;sp?RFcKp+Kz*#{2P_SEjdcM{?{aP{2@!GG zDpvXG@7Nv@OiDV?MD*R2@JiBTW#felcg9N@p4tb-`!N%#kdmMD1fiN^m_ zbv(X@i(i{&Y(!;uruteiaF1H_Oxucd3Cy7eb_`A^5x{&8pIPiFbfN3>%2)gXQW3)_ z?ykrhaYDAAxVGTtySa-OxSu-ui^b?muH7H*M46L5oz(c>uZPcZer+eVb{>$Av8yZB zujdbQ_OWTTm2HHrP94+bm8aj>?b2S=cbk{6)=sBkwq6uClI(TTcs)QO>fLb>{vyTO zAH+j0pC?cLEK*DLzDYof2RnOY1mYq0J1tZ_%y?6#SLU=+T^Jyb`eQdB+h8kZo<9#kIK|s;C4* zUW)r!4SUEAFY7=&pP5NHTJ_P}zj(9jN3AR;1?MbaOHzCDA=tIy+rM>)@BpjfEpTfH z-pNk{z{i3jpWD(tN;J6ISAL*(nAywKlI&|VN)&5EdcBi)h^Rd-|FYKD`VY`$z_~LGa@{6TzYV}j$i#POW5OGjYk(<&@N1pNf+a4#4dRLtN3}llOgOnQO4?nnuR}4 z`zMnxuJ!DeBVqQ!p-btT3{hTCz$lO3+kJdwur~YINerl;n`xxGXQ!_0rBj?{qJ96P?%kG0s>%b|qA!;TBsF*L0{} zs(pbP+I2*Q*WlpOu9nb^@x1oLNtYdmoL{h}vC-+ahH7H~ilCN5Tzb z+PaVs5;^n2Br49(+^AU=DPUYVVwum_&8a$};Uk_}5#u%eIlIjGs_altHZ`C{W2L%R^W1Yo@zqpR(*Z3J+*irOF?u9 z(H1YIcg*dtf|ZjZq?8cgOYI-TZ~gLaL~5RW%AV=gL{6==f>pP8s<`>)_DkGNV?+nO z4Cbmt!?39an;2`E;4Q$$g*UPKpCu*jn$nq}#VB+y9@VEw_^i{;u#jHQ8AW09ZZJF5 z-k;Zoe`Au?`taUjd>r*VI>eawW<&F{Rzm->gQO+KL!6D8lG5w9q54E=6}m)2&5!KT zJ5>G6yn}nyz}@B$F&fp(o@PN}sys#SV(Vw=^EUD{^6X1dDuQ@mxTx})FH=Bs{p(bt zg}gOoWwjVg-;x*=)jFj zXrcob{!w}PCWIKCiLr9fD%7E8mf^&f31@7?tm0`X9Pp9HbJ*JJS8A;RSFdjT$cGad zKEC}MSlK)AD0|xp)qF3->KI#jw~v+6-Tv#|;O=SQjtWw1ay#~gPrjOX^%);wC7)J= zdrfVSEeKWnVhZZgY$}`ftEp{CTZ!srk?7B9zX5Zc3C5E=OOdFf-h!e(S2|GQI&YKGYza_{kH*c5xDqMD-S%wNq z6fV+Z*^+qeeYH)rHc7iT?$=KY%9@7L6Nc7PWtX%u-}NRR?8dT4Y*W0-9s8`P8Y8nf z%O#v{2Jiy7z@u-aF`Xc9wptmci}0*C7>3J!Ua`;73d(`n&5>y<@nK5NNRt}-98b*E z)q9utHP&AkhViJn!chV>Cs}Td@5N6%9$X6YiF3^-UOY?>sS~~khb#f??52p7j0-+d zU5&mm%B7DGKCR^*wsy8IkDH+kV?wZJQ_nvT}oYIeX)N;@?dSlb_$jq@Shn zjzwc_$aFD&$Udf>ts^2z8V^$9l4Bdt4bBz?mJ`6_jmEBl>Ax zp3m5BK{lOc6$3NGq5;dxH1D>V!z9FC#0k9dK~*URST_gXO?u7!1RtNWoHOj zwgQ)%9O6hFIT`%I8Lll+bH>8WP#0J1TbaB3+z$R;!zmh1LUsu!!0bFaS1WJi-*+u2nNs0?E(lE#1|sw=0f*I>+T+`kW>+QBz5;BzCNJSQ!0DPt{#$OH)xo*+uHn zLY!Tw&q)~dQmr;x`0({d_s6vs^LJvIlNmNFXiZsI4j&SchPSH)weuWMvKrD@gM|km z9`4;>NM@D(icFgtM}pW9(_|BxS{Vb=*J#4NRUC`FbzKtDrar`f0Nl_2vrgsM0$Ki< z_q&~=cp+>k5D6Qu`s0zt8PbZ$#z2oKA=$1^ro^zqrYYj;4VldXqj)LacCRdj;{~YC zqw-d|%g?HoYZ3WNiUlkH1ZL`y`2B}O_7NK-ecj(99Q+9yEHGv-B>4^=+Y_J=)AS}F zY-C<{)t^!0R!jn(Ngt~)V>wd< zK>3rwVV{OiKYtg`HZc%xxa3&TsyC}}z)CEU_F9ccX+oFT%Ri-`6%Uox3NIT6J;WZW ze)c5$n#IB?!5qz5LG5b#s7Ak6Wy4?E+eaT0HMMfo8qf6 z>C?;aVG+4;yQ2bvnbih;ovoESV9~_lBrNvC>BWZWa4UwTw4cHMF^};t4TS`>dNxy! zDjBxuhB5h?oWc-i1*fIOOt}F|bk7h_U4rBlYTANy3H5BfPhToYp(4zO^ln=zd zHfS5l2^4DyqD*L+LW4g~f%Qfth`kS~-8^|xQBj*=efDFPNO7SFc*&uz8?*EUcHe60ga2(-d78 z!f3mBR1}83oEFNn#qU;AmZ1aQ{oQ}rHn>hooMQ4Kz-{eN7%5BNsh53Xy($H*V7+!& zPng^j+lY}<*quD0QX{>2VJhJPzT25sN4CFxx3C*dSM}Yt?PC8IY*d!fAJrhD-!%^_ zJY6b!+`i}A%q3r7sTMKfo{Xao$F$Hb1mcOxe`J4VWNKEf*946q$xz&}B-zS*jd<{mW43ne5jj;y8c9CR9E#~bJ@>91Dg?jMchLKtqWnh~+;irV0JKT5c4 zNp0*ScYg#j>FZT5_&lTr-TPNAaV5>QL4=vm4%;fC&N)rNrTRgJsBf23Rl#Kp?4&9J zcm+u~)=(XTH*F)lx4FkH1_>u(5Nj<4L?9#RrV!tvF~Ib0v!om6v`(p$M6E}%B2shI!m6xiLw24 z`6=ciywt~r?}YcMn@oT5R_sr|jw&mKROfcM2wC&CtrUE=WtY#yG2;sh#WQ!sXE>I- zBFi1Tso>vZ8r%_~?MLP8T2dkjics6L>6>(^RIWW#?Ql@!nnBsN;MTWYYtHqYYf1reS~DsAEH~8CyIo<&RKTq zt@b2$@* zu1-*Vk#m_MgbJ-*Yh*6UT7}K12O}5!;KfLUHuwk>`q>YyPLy}!%PGqu!}wHnnDk|T}T`aO^(^@3p&W&Wuxn$|3V; z|NIEEcvk1C^rlX{M*?r6aUHXeZp*8SbD2&93ufS!tR5n|WIpe1-K!)>&9-f5DS--7 zN}Q-YX|GsD%%RT^ zI~{-5XRU>vyGJpPe>L=FVo)idtL!YC%G8Ki*1u$_cu5nuAas7m2P+%Pw~PW4&i`AIaS!J{E=X(46pgS)dR#seS8c!Or zfLMuXeL#FCWTiZzxBU;W$<;t{bu6;iwXG~>;}$8>;S@B0pZ99WD#8WlE;z}H?CYMP z2gC;8tq}R7pqC3p^&q*ESgR_Nm=3QWQ1hh)i$_VF%hX@4?*(+ad`W8(Gmr8?hf0|| zbU9p%1$ly}YlXi2qB5+!ccu2rfsx+0nb&F^kwLH3KQ#pheG*QzR~0iA$drC2O)Ds| zV2){B!_!)v-F*#wRM`=Ss(;N*-t6Nd$J~ns3dnzUWDZRn0GQ7)BCuQ~Z|>SapzW7)Uf1S}|6R`0l&y{?LM3|^^1Rd6Ec4SOeD)o;t{j^>)^wdA$;o2eu4zR{*ifwsi$wO=;1 z&}H{g0jH?ELN+h?EG?Ot6Dm#VFT|{V{(g-6K-WW7Y^`<0_TO*(O%>r>t#|L?KEH%W zhY8nOK7rF@1F7FUwKS)9Cp$d%GMVh;MM%JUmx?ppIlVSx)x*qJG{zEL!Oex zbAM+02iVVj)c{epGD_cfCOf&z8@yinF)GP4%hI`t@4pawgTc0R^3Lu-)%bA4e0C&R z=4xpNe_-GI^$L8m>=$s~&$~xX*+i1WjZx-Kaju?ZAEqXAwZYftl?9_8SxJQRIv8EA z2u=S0c$&a>Kyrh~P@*a|xI>==4gyFZH~pWt@Q+s!XD20T!EDK!h}S0mzkXcfRa^*{Fi0?)4WXAdsn~O-{$ibhHWB?IzLTa zjhMbcq|25ifzBiE1UUW(XO=%DTv{5;^z*Z|6T+{{UOXh{MOTBAb%m{{bd@ z;61<}{uk*}Bb(X{s_@Ew!1Mn9E-7ko<;dv3TDXfHp!O?wLkd|AC;5*t9;F<>3XidX z$v+}5VE_$9wkSo@37;d1!`E9SdC0yS8DQ^J?*_koG`sbCaA6!%jbcBq3hJhcNxT~k z$H%$cY&Vu~T4MDzQbIzEwHc&Pyht3W%f@*M39Tu;yQ$wm3k9FBqEEg*lNwiKk4?hy z=%cFli#Mj6BTg9IerC@(5)d|P37(k$Lsl}|&C&9wuboWMwMCrhc{ zBA$_fV)d?d&a2iw*F9ku5Wx1*UU6H5I) zN?Y*bkpp)t(Pu{AWD7H}1%!*F4!B3S8kfX7ICxvOMtrq>sr1{NSHih;{erppMW!q$ zBDKAq%EqAcY!^dXt^>+CR&j&EAWhxce5<~qUE`55ubjDCGs^;7w#NNxzPV{<>I{KH zJIrPD@+s1W{Fnt?^k*KB;bWQy_bQUPm^r+Z3CF8*Gsm!DDQEASy9%Lvpm!QSNI7MI zuC@b(A_V&c!>Mv0!EZOpz;q!|^)|w`#1>2SMiBPd4V=|;*QB`QR^q@%rsZ}uu^*+b zdDxkKaZ4Z(E%RL#pLZ&AVN+8|R=B%O68|SbwJ8csn^S0di^!^xn96bCDYEk%S%GI8 zVXSd0S*g&{@@;9|ey}<(S_k^)XI82fwYqe4} zyRiVwTQ;|q#;TfAc?Sy!uFls^xg}Gw&Qnj$RIh?|s$uUEC`+3g!8XOFdfp75?no&t zw_thbX~tlOsIdScGGfP@(Lk-)z*sI0jTI3=qs@wlxL-H0m?jaOhiHg2 zop&gVWnQfGYI7^cgQPE_D>F&ek5t1e2~n0-O1({6?ki%kSL3~EDu1euBb(~4x+$C$ zC~$`{JG6J$)09q^sQ;fnkJ!$FAncvh(&UHVmK=?`(3<`J0Q_V}339DGPGO;cO zvuEXP`v+BIg##O-a%RR=?5?|r9==*hpEiDm3@zJIll~4roN%dfcY}33r{n27PQ{IB54(I_8%|L?W z&Z8)BzxREYTMWMOFTh&R^0AC5T9o;3qqy7UMVHO7!66Q_m6qH@M`JFARrt4 zZjN{w!D>hqCIJq4!Ig4%$;OcPm0Q|H9H3-6!cwj(aK!SX&Gg+{ZbbQMe$wqP#MpAs zQ207daACe7_PldsO0jlNG3?JzTw?BrxAP-DU~q7t0HB|rjST;K^Z+s|GAqPgBi4$r zYn_!g_MDi?FsnVXG=*n%hcV==Bwn-s32H(D!o#I&tXkC4``+>Vr zL#AB7YL*#lX`5jE6!rA&*a;E)<>vM29(7>`5b6tM-SE1O;8{tDFTSW+suZlbvPaLG z#E#cy*_3!2kFS_ed^Hjyqw?nN*x<^omNPg_V|kY&qOs?F1jIr!if?PFzhY$|QC~_5 zNVLq@KW=k|PiB`fb3uz?NvkHE4KFhfj;VrU2;5TfMR)X~dJn5t#CAC=c+K3LmvX+$Ln}6IXE@_<_)LBiIRrj|4Xlx2GU zp#53CDOaGRPl~_RsqA=i)S_L-!v7|MEhP43`jsog^#I~wY-UPx5uUevZT@yp(B*LQ zTDMtlsdq{atj`Ou{;O^0gryc?QS? zGGftu)NW2U6`VC1={T~~43wKOViiABdrFcHOTip;J>0)m{A|Ya2J0T@-KG3C$zgYu z*Zp4X1_o9`g{Alq#IRJRF2Wtm%oj=fWiY(voum|&;yC`5wOM@rf6Jq~k1G&Gomu9+ zhTlDcin!rDOz7BpQoRc1slV%_;C{UF6@*xI<@&f~S==LbQ^%eQzWiGTt4de!e9)~& z{bjK{jXJKv5UFus(9?XL7=7C`zSQ^6ku(%ex^yN6nYY3oeR3nA#sVMJoUpgDA^`Rb z8!8AU8OQ(w=!x3X`AGbzQz?NPA{}D&Qv$=2&|v+{W_;(SBO!UF2P_7I)>b}wWQ`lf zXORM74yU&Gq?7~UBWRz0@0wDm~Z!A-Aqm5Jb88uF^pZY9Afmp0cHD9K_Nd{F!a$GDu~UIOb?C<1DLnNxRHjW zG%{PSA=K~_f}cF~jx@SG6|bZgwvQj?3fkxOEQ6P>j+E0_dXP>N@&Mjv{{UzqyS_xB zbwzk?^@1I7C5$~tq{PVqaG+BsBk{;KdxNr1w(&RiX^Gn;vi1Buj$bQWZDxFh4|A)W z>62Kw|8F`_gQEch`4qQfesPuC;(Gth3fRBtX)@l-%xQ<)6JrqnAnF-J3px;(L=mp3 zYS@aDm&AnfioQ7q=WM16-BW4p52-D}p`A-uohTV*mIGOaT-u=62*sAr#I}0ea2KuE zX%i7_LQd3>0Ie)2`34E}sEHS$*p83bi5Gh$GX{_sTJc4S%u<5C?x`j&9?yh0Dfk=p zf^-r^sm6&iys`Y3m#3MAS3^8o(^*pdBIV)3SvIDWC(({TtZ-=1qtlDEMR?0LQ-&F7 zY_D1Dt^&pvtRtxPJrV_&htOQ;ZTUiS(tZKu7i+x$Xcb+ZHd}8(B9iSb8fcNI4Yr?igpegl^#U@gDdgt#xnEEJI}m5`iSM)hYvCRhF|H}6^2M*4TbEOR zK%~tYH~9&!#SWIZQsDjrSgcw-`NW}wBLsp1osUq2Lt3rmZq7O_xNy|!XyJ>BFAiHN zQgv5h-p<(+{OrGZCzL#3LPu4xx12MsIu%>!3Hq$_=wGG$ewW?Fr&POt8)os5xhip_ zb>v5(zAeZw*%CW|2a$3M@j0Fod__GZ!<(jR14|(%oR#uOaLfH#^+m_CK1s>&(l-=| zk*lDa3B6qiZGzk0%p7O_6SnMY^5(TAPHwxB0qM4T)m6D{lO%_UxL4GEacT7DiPMiW zN)D-UK5r|BTM+3=O4E;X3e942mpB`mQ&x+#zBUbo4#%$}u1EAw1H8kovpVx)K?c`B zsVdFLc&UcUs>Kq% zGBGI3v2mYohAIWCI&4Es?-Q7ljUEU=0%vfE=jdn4lg4+ZABM3ZaKsBQ zMV6fYk0wc|vO)r~EKMWp&XV=Ldnm=R(Jn=Ry&o$BZkkAR{kC`El1+6Fjmg_ywf+Yh z^ovDO>~Rssy27;mY5VemCXc-4=&5i1#yGq^Ur-OEyo35&rXeYO`-&B+e*(fvu1msj z&6oacz&$DU5>w>U`PbRY6i@C(yHb(=08}qUB3EZ!!!_P%LKAOYU$*_){W2AXp5a?) zc);ah=}#z2S(Xi-J_MKaT0;G)-l>NbBd{ckq>938CPlxdGzzQQ1{?kfxLa0&mw81v2m`xYnr6Ff0bVqH_rk2-2D|g5h zh~GGt&wN8ycdd7Y&R!-X13H}8*4Vy8$-&G0ld^~FLtwlpqHHtH2IL$w=LB1&%Xdwi zNV1Q9auMOQ=aVAyqt@J<0V-uTIpTl8CihWrmb($zKh!uJV#*aN{~k@Ey_DQpWCLVL zksXCsI!HL_F89?*ycvYq<^+zfy+Br-b_D&n$JvVt-`*C>4lSsIqwF2wm{yong(-4A z{SfhH{)96>R)PGM2%3S_JnO{(O^TtGf~94hU$kAx6$B_(m(>eGW?b=%k&FY##fb_X z=HcH==BqE&f02-Nr-=!R+IO2d;7}|RKw~}H^H6w3tJ$5@)jP!YnT?LjMn#w6P#d8* zTR*TduR~lr?fSo~6IFQjgFMb}p6gY-E&3fu62L;s5RkS-!tlkV1)tnZHBQLaR>@y* zzI$nX@pfjKmCBKoC|B1lGv^QU8=;stus&-av5Q(8H*)iIu8y8mkyh1EKbD$|m>+Z=V{nmFT-^w-Zuh z(Da?~C>2^>g*~$GUnyoLRqGlEAnOVdoAt30CuB{-)CVLpDpo%);a8EbD6KNR)mx&}D#+`E7NGwF>W@s^@3Nq?4A+N(-+ zP@og!1(nNEZhEWfuUOmw^?{P~W(*|Be|U!3;oKhN){PPRez*CPHfgN0GpZg`+r4?k zjni3hGk}PZplcdzthgcX?tt?acwa9Y^7wk8mVF&t;bw%n`412-+rmcAp0IoeTc8Wb z`XNdd{NKch5Fkp<0W`mgia0%mePX9yrJsOy`nGgN-6TdFIZrKaxKxh$F!#Lb{Y(~Y z5MXb~S=oV(i1ZkOJ>0FH{{ae=>&jqx`$a3eRMk69hF%Byevvo%hnlad&v3BB!U#+M zNbRpcKe4UT70Q9{gDHCfGNw%UttZV=2Ie?JAfBh#q|f1{?A`S-ClR>TrbFi zV!I51^s^soS|u7K`VlgI`UN}HK4hs**GCR5W-4s^Ah#^hh_DK$Rmo-I5q%e^%baPg z<0UzEZR&#HnRN3JIeTx+uoJM?xG~EP(X~|G)ad_6D9dz@q?s{;VR03VQV-RI4(?4Df6m zQP$p)BO%biN&A%ZAcf7fSjMDj9PcB!S+6Ua5WjgWBA30BxNIFKrYA+_cZZviewSd4 zkH5lE5vwU)ynL1ngBvCfIZw$61w|}9Nu#dT48K)k3X5VoX>g7lml6I#i*JtEFu;x5 zW<^=CtcHh?J0)l_O0uiE3k~%7iDz!w+kb#@= z7{d*1?FXa<^%+){u^~+MTa(;x|Dwi8dr@okIPv+_Caf}*PS>krp!y!W7P-pHy6CYs}9%?`2AZt$uva7y91;tG6*jBBJsyT(ek1=v3*E?2P1EIa|zwm5lAkX_R$Q^%OUIs)3&E@+_`I-8~yI0#tX>x zNUOJ2TFw93!{oxXY^|v+uRfm43(Y()ia*d!X=MLu8|lO_h$ABQFh`!cCC}7Sz{6Uj zS973@HZb!RKjDLdT|!gIu}%-D?Wm;V!^kWHZT(3j3?ZS_@tE*e`kcX zWWbzCve)|NyX59aqJAN-j4ZEVgW3eybUHQ0L(*RScMig$qY;i~{cdnCcZFeE0Y`YR z!*1J&lxla>M&r+pXV9Sm*Y@z@foc}Jv}8`76RYv`4-CevwefA|e!n}57i3%OCz1iv zkDQIrW#A(&zQ-TfTZ$N_@g#jqn*_;YMHDV4p&qW)LAY{~K=$y2zVQ>G*7cL6d4Mp4 zCE&RU-`*{@@~fo9*05SjjaT`vFKN|~Q8ojE*0~&WR^;i4cLxS%+v{_&mc9}Djr%#) z1Z)HEb=JsgMuue<_2*iIczRV8{_3Wn2&MmCJ`(WtOXj`qQ3_zy^-5|tEs`%f3eI$_ z!}$}F-nPBmwQcA-mxbR^(C{nI_T^wEqu#Ra)9a^GLygh_3y@Y3T`BJ|1<}!*mb@kD z@owp<;Yb3qnGL`BTD%v~ftr2;Rz%8si#L~tTGG4wIghkN3&F_%Mf(%>Y+O>{4g!BI zj(-b4uBO(%#mLlIta{uRyB!6XpuhbE zj%j%Z-)>CSP(cmPDD_BLl($NJ3cd>=p(Nq~gVe~6R%pE0AcSP<)I(k`|xh4Qo9yP06P9<2@5+1KrAtE zHi=EKjhG)_`)j-H=Ik_U%H0x6%5(;MOxQobhHc*Eui%?^P;o+^0_Dz4Lrhf%iG%`S zP0$#EzZg>-7PXL#VSSX08QS=0S@p;6m|4~(6wP>HsVCyvp{pin;sUF;W*5w>2cB4+ ze^iC5Fy!!o6dti?uGq=TioOTS0rcbmIpu+N+vZ5M%GiW(HzwAOqF#&PB)7i(SI|v1 zx@P!^Sd*<~OMvuQdoB;-(+8*NZ0d&Z-Z?;YsHuRjaMGJJ(s-es{zjk~nuTpz`u_Wj zAI=^Vk9dN_E^13kt^Oa!T$T$AUe+Zk#9pirvcg=27>9e868)*WOEXMk*bY^W)^MgB zOhwvKus(^2Cg6!3uuYTC{P{bbmoRrle^PmBDVs{K8^L;Z9G_v?MEK-V==tF=zWocu z@Tw9=+hsNei`3k{=QVRqBlB0c4Ex)|e}K6{iQ>Y%BH0h_?u`m(pakiz=D~bqM648P zu`z9)Z^c8$gW2XO(9HnKhG)5OR?0+E-aji7FYi=!Aq+Et55}UENNZ*akUSie*Fbo= z@2Q)En+kRknG13H5zM_>&aMxA6J8arg(> z02O)!g*U3vGaH(TTXaufl$C_NycqN zKBU_hkj~6cs8bS=qYclVAZUQc+Rxh53HV6|RD99MX57C`yh( z^g6n72D9dVgJkkdk~l*knu3v<&&wbRb(Os~HF;C)-PL%L6*&ePRu81xLJNN(3aL(c z5fuH^?84M~Lz2m3WPr=uR{M}%&Sk~%{ z>m!j;F%EfQnh>Pe3WISKw?licB&q!WDpRjjSA@Fk6HP}yCzXQw`W2q=W%B~@Q?zxq zC09lV09bk#>!d!CiM?{nc8GtC0jyXU9Ny1Za_3PybCKtKEd*Mc>D~Rus_!|L7om&q6esjqMJZL@Md#sFb#O+%EX!f<22WviIvn7c{JMCfEt{s>qrus%R*21 zg6R0_nznBUm%(ehgf4X@$JutHz~2iKqWR?0{_-5Sq&$_>buTk>Z&z#nq42MCS>1sU<9i@N|Hxl!+fsNz zaS*+y0!xfvsBm|yN!~k#2?uJhcEWDRe)^GKX8SWnM|D8bkBAz9m&=8@{+10s%HXq` z$Q*Bk=zi+nY;UZsD!ml$r21d6{2rcF;iNnK`SIL8uOMjf^v>tEl?~zE9GR0bdoPe$ zBW2&uY0+h52Pbb_er^jxk|$!=1;{Z#V^Vq*MAPqhm(w*|zne$sF(${DeeJfQu>BTqqmF~(NuanbIDUo&xrdTJP=H@o`naav=^Td(u za?7FyZ43H7k~0Wo=7fN3kMKA9RRg|)m*fhwh{T;k$;s{1xXN8^g*VQlMS+Od%gUnn+zyj?Ut>^(M^&t7Rm_N{z zy{TAqj{z<#Nr;Uy4SXln8VpLNw?_*h{)}Nq8zBL4-=Yi&Mi+yU>9V=3Bu*8-RuAp5 z%YQPb;Cu8xgglf8X4wpNv#j+Wb$qLEju6Mj=OWZ3)x5-N(37>ZWZ)ARoT_X8Wo7Zs zoMiK7lo!KctbIdl<)OQmZ};A}tD;7PO;pxUK7*f@BEc?FutpP~5}UCy?`Lk)R*k=Q zw`9ke8v3D_byWUu2qzxqFf|=6UO#{N0s99C5_reeL7}WOdY|%Y5Q-*oYHT|W^!W@b zIUd5O6VM-)N!g)X##d7Inj0;4?ns-oZJkRkv&ElFaZWRjWN8!9e$+hf3Yk1e=qb@% zQv^{Vkjem%$ea7j`Q_#<+RvCIEaf{L5@E%MdEE)?U9*O$}TR1;`VLS}ZGj`Mr z7_7jLRnIE?r5(@YU`>6szZyqyc6vi0H#$v6X zkK7fJoBVp=ajFA`;5%->UhhX%GowYR<5uWy6%xOL4 zPs9B2ya1RSlORn_{c3rg3Ci`vY6uUOji~B;KD1X@thy~3MG4;-#n%baP~(k~MMAhl z_^QD?3WG2N?3Vy-PBhi5;_IFp(_+atoU~w)?nTShF>{>qg(QV+S;?s8Hxm_a~In)IfESsVmRH? zlqlfK&Ewoz_jl7Y0dJn1Y}CW~RBrY1IVjNnNEuTILkt@&>@`XZ*7Y>=X4U#TGx%LH z>+g>CEZ3}Z%gJZ&c@A@+tKca{MSILC`HR2X?cZ0IH^=z%P6oyFUUR4+r(E*X#H;1P z^7NZVdr?o=ldrh`+b1x|;v}?rd#B@w{R)f2F%Re5SV@Hw0=+H~x80#KFnKZ&l=$v& zPUNfOs3V@QwFLT3r8-!er~IKw6T73_ojE6j$bbL3*cE8mjlZWEaj%QY)VmLBNhz9(&t6&V(-##>X|rt7^+t$M!Et|Y0k4^Kv~cRYLEIz*vnS0F z!IVtxv(kX3d2m4qPAUU(!)`D3hy(69DJXvL7w;OXsBcS&1GVJa;Q$F%DO6lAsksx} zXqU2vAN-k?cBj&NmXs6$q_&mb#v)sR?8e#`t1iW%03bGE@gE|@eX9U1C2-6(OAHVN zyeKN}v`KW}*2&T(IZ+FFTZ9;I(#W%_E+ znBZ*PLk8MQ5Ve8aL;v<4#0%US-e0*%UcqP>!&mI*tRGdu7_u2H77VP?(#P-~rew&! zML7mo3q6;HV0F#4ke19A=H6Hxs%G-Fu&4{Lu$+GX8cWO`KS*8Scn5~f;dnZ`31J~= zqhBQRNXwO>=2`jy3!;TzwUIjN$68Td_eo2HvtN_bEw_ESSOUj7rR)Wh)pTPWisl+~ zv{dguQ?w9j7Os_KDm9=Pc}T$*`)oNr#YD)SJDT#VUU0Z-(O~3b4|(nuPmF3Ppkz>Z z1MNIx`|Wpc(#fcTFCfqe_53+yGEE~`gZ^()9pVcUT$xP3n7cpZls@BU6~k*{C8^0g zzUQ-D4*z1m*P2A1V9B~Hg`!;GnspJx1A{faJact5hJwDBazro~R$nlPTwSc+pG=)@ zu6*yeq_bujE@=n>p$XVqj3Zo*{M5AVYjyca5%J#QcikFURdWAA07>lj^mVDGDNE3z zEI#vTxhodBVA;OZ#rSc_W$(GtOgZ${N38VkoniUjx2UkE*?|vv1VkU< z+kYqJ-r%sH2%Lir&+DBMV|NV=qv(qP2+fAllP4?IBXNWzXd!NHz8K<&n|Y|qevL`N zO-TQejb(w#6XTEaxiiKlEJrzbpZlrrBBrrzapK8;PUH&z_%P#_pi}|;katLYi;Hx! zj463m6Jb38tA@`y`HvL`1|(HFFs+6J_AQIVl@p~Wz601ZV#8c zSk6MW;+a6lFy}tBZVcpU7L_Ax`Uy9YAFL3sR24sG^)V~4XsLEz(FsXAo3gDKcBt}F zwpmhrhvu0B_~&-5rp;e9YF#^ApE$9k{-B>u_2n)3k&vjeE4)_So$0Z z@pj87iil>Ob#T|wa_E}FP*$4O&T5_vYc$g>&R!0=#j~%iRV#FydyB6-iB~LJ++XETmtAoj*V6^Bd{YNb`q+3`csZEb=t}anwk>SFj#t)7@%RZKtzKX7>sQsMEi;vt z=OXaD8co$7lqQqprI4mWa=e#9tH3lcJhAaa*~#Kr-uUXt za>=HryAw;yB7L$s$bWa{nY}(TIy!6$MRMZOH(VD=fowJd4S_~5hnG*TAx;9lUz;4YG z9v4evv7kLN>5nsm_Qn1jE~t}h%2bsQRzOWW#<&_i?4E&hvd9%0uBlUl54B+cfo4jc z?C+{G=E_lT6$Pw5GVEh0*Gd?09h9EpF9xP*idnBM9AjA}q#yzrr!E?4uwoXHIHOiotE z;t=A3#Q@P^v$ah8OtBu{_Pa}9tnKUDO+HTd*MB2>TZ!84BCcj_y9bwewbcoHrz&b?Z~tPahmk!^=Sp_4I>|cHL5-C>XBpL>6PWC+Xi( zqorXXM$iYlZl`%SxFdtsz^%#Y`8TkfCf1HLg=1l|d)HN6zKM$x@rkwQ?%^g(^lE+W z#F11-qFvYoKjT|7dt7d96o^*p3o&oWt@ovF7_B-}~9kyL_zFMPq;+ME9i zZ$&Sb>^QCiGl8{aZV3g3=(k4X4DGWmB^c@N$}#6dUEkiHwX~y&R^k5u&kD>ypx(9@ zGd_@Y{6cM!q5#$jZfR2{r`Pn)Fc1mX-J%~nXd=keek6UB!j`#5R3del-yJxR)&>=2 z4cb%Y*>CohRQPlp^@>M%ZUE!u3oSL9esqrVBBMvy4z$}jt#j7(Pr+}-#v;2;AbGcy z@2-PiRk*Wwk$b`JO29G`N8H02h|9wlef3dN)WR=8`7md}r=Lit-~5+VQmZ zaOV{{p)$~G$9K>zP}bcPzXZINIT>!}elYsx)~Q4i4g4|;R6i27wDx=->z-vGO(H`1 z&hZ#gdBuri6-q5A<_v@@Z{y%jyh-{8X#Ir8qJ5}-%)@_ZOPfRv&-3ZH>ABip89k{qVl`0GUq0_72SK-jDn9i|p zbZWm|(iHi)4dE^_(tK#P-s}^G5S0{hLJ0mCtHxma*_ThiVHSN}@|ib;a`7Z?JUIp= z6590Va+`K7A7KK61MS!SW$pK$4Q_W}f->o}=<1&R<$<|<2e*4$)r2iv!|W-+>j;R9 z!xY{AHjhObUS&6re(vJZ;e-J#dO zg-det1lN?-TY4L!>0UDuk43IwzF8gDUfT`@550}#w%*#RFlYLc?%GFqV7I3!_a`Zt zmmRAp-*fedypN~4$kSn6T7{H)j^3Yn2KYSm@&&snb16HMUqRbyKPiJV;Z9Y1;gRvb zL<>@q5xLhhn|kEfoUBgyiK&xE5a&1I&hHXl<9!h!QIK$4#&7lZH5k~GaKMa4%^2&x zUU6Pbe^2OKeb!n$GG9VGTe9V?o?fHHqC1V!Ivo9tadKaPHsl=!Yky(|!Qxi2$+9+x z@7_s)4V3ky#>qztm|KUOBv6e6EPxBV{<9!7$s{}WW(seCXKGnZ?oVfx$>Y|?N^#%M zi4V@726ZSiDW=BtiZd(cwuNX7uNQ9eMK_UzlYeX}Y8bH?i1??N0%oz}URmXjhs0&& zKUW+jq)(K>JcDHT1?4l#VZABg@7|>S1IT35a@F|50K<(JZFpL9DGL%!=N)?ebnEp& z$Cl#ir{7o-Go~j$S~vM#BOxIBDkZPVk*&LDBfh5^q*E)pWXT6v4jWe7(LsMvA|fzS zz&VAAY^%2lpZkb;L)4lY)**07c~`&0*3|a-KR`o$2xsvV27k1?#y$?hU-)RIW4^oV zr_WFL^rz9(4+F?q^NMoisz`5L>ACz5egK^9w*W`^lXc}+?0cUEBgDIZfTR1$Kg~tL zKk&DL*Dg5LYTwi)qLTme;*6G`m5o~FzksrePRjd>n?L=W5hD1g=vDVF1u!}d?VlWK zKkuJz*U40=wr=HJU-U(W1)%FbBzF~W9m%2J%5v<+N+>6=HSZ91wXRpGWN$^rvl`-` zC&b479m(wQ<(ly4sgaF#EJ5b*bH4Km>j?xVflf+SUg3p0qc>ueQSFjspy#6v7H^n~Bq zd)Eeu*#z$E+1A2KHF;wyG($^tuTrkmStb@0mw%lG|{hozZ-6HRn^iG#|KH=LS?%dlUPnA|p&~4yD zNvgJ;u#nlED}2WyUuZ&Yvok{`?DTf}Z>va9p}J#h_+W{zP-hA^qAvB_foHn7VXlaF zUzwTptCyLDoQpEY-gZW2Mz58;(k|)uEKB8iuW^c}4?JmNcs9=#2c$hSX)^s_@ve9t zit^#wehbcL=MUgDJ)m}O4d-2+W)+)_K+#b>a$U!^mm7Qzh#hiCBKg1s1K2c6XfQv1Ne8-&X_IL zlr;$)YN1sU{F%#2LiYIFgPi(g;K-sO&i@e{le41>;?{cGfGxc8p zp?QneY+Y^-3K%>)iTW=qo9sF0V-^pUtGJaK0azOpr?E$tEPA3@kC_IQV@m@XWPTo| z!{(gBPeRB_Yh)a^i*~@hBJJtMqwlM^6DV{2XrMUFtu86%ffua`-E4+aWqvkK0s1&w z3V31D>qGQMLek`2v9J0@#h07KPS*Qp(q;Bhw+1Kh4FyTNQU%(NGE$`l&&fj$!E4&H zurZlq)UD!4@s*AK1if%L&*KPdT%}_}!0a_V9V({y(n527D{pbK-;(H*AmDR7C@esM zAaS0nNPS_ExgfaKT=&-Fimp-OrW+mA$2eC|{{X-fxBz34^9-ug71DW!?i(j)KUk+l zZtxbix_F!Y1Jnlr?^)S&Oc^y%r4~zh-QU~v1_Msb> zvarJtX?wo8mdmCy;l9V}EsRGx4=^{^&5-l2l zDLT~7!>+}9L1l0H>ylK|Sn8OcOKI?sEA>uNa_GYnpzNy^KH9Y^^uxB`6FRO@osR$a z6ASWQiRUQ2zfy|gXhiAvl_g+0_OO;f#ONWMeo@8mTeV9hrEKn8sACRPH_CZin&UZ& zOkT}cwuCcoZ)FMcWnQn>Sj&8!2OUY1)PUDFX#S(QiSs=p>0!R!l@&UB`w@`}%#3&WXf0DVXVKggR3 zItA85!v#^}`V1RizPmWzknU`r(#E=;4FCEEs6z|5{wh7N9%WnkdukgVT{LIF{95lUAv2WAi<_XNfJy_)tl#!7?R_)_pO8a}sfc@|uoXhfVBdb0%^ zDTV#DHMir#!JIWlHD#jcio{yK@AHiLxD$z&G=h~CL)U#AVI?QSzwcyVH!0<&)V|FJ zIzdvK9YF|f#R@aj=N~t%+{Rlcj}3Ra=<`OJQ+ZpuaetWLVa^tY3*=zjB)g#{9IfP; z*IiM$ixx@}!0Yf%^2h&xvW4C<;S?`}*OG0J(9{@`ne4(oV7S@h!4iPZon|Ud=*%2oC!xbU^Wwif z7F&h`qoxey?=+MKG@s+9%|xIjo_7(!&m=24cJF7J1Rn7ohKXG!izKs6@oF-}upoai!Fb2x2{YJU3dni(+G9astX+MZl$j?Tq{3Dozr~K}j z)<3|?Oipw#r-;Ge%a)`EGeq#ipZ|YrY#Krt)mw!mBFqCuH@$Ef2N!Gfh_*$`xcsS! zREUv|N6YQ%M#oqOM6L&wQwQ0y^%+0ruV;cl^O8spywkh%9Dz0K!6QFzYqzST6^TmG zFj^ndR8q=x?_8)A!stz(V_=l#w}L6nc-Y@F1bF9-G5R6shd$Z&a*MStkQE;LeO`u{ zocqDFkNEjaH0QBV_CAd*#$kPS%H5lHJ(0qD(olP z8@%B%znw`8jYV$l8qaogd-L8cZrV(cfFehpXN%4(%uv93;L!Pxb7sx>l9fT2wjTCITA`$NYsVkir`om)Y6bw`FfK^bc!kWEbip2XEDId#?WUPQq2| zV)1Yv_sES~!{Y^@8oIn+wBI!94g`FcE5!7dqA|(v!tOjROrH)p+D*gXP02H#RH0zg z^xm=1A|8mxMH`qCs`MC>uR#OT^Q+QF3MH9M?EexTRWjFqOsEszIm~;$bmcxh*ejS`#`a> z4OUp0xyZ9-u1ei@GvK^VQf%bG`WV&+3l6wMn^8~YIo2iotfx^Pe*H}yyWsfQ;uO=3 z{B3W+%`_b%-;Qy&nYV>rpn6KQr)(s({(Pwt1=FcqqaeCH{yk0c6iR+Lsv_dJqX@Iw z%aT;5DYmumI;}h_kawbYzonRG>r$tu!Io$Bs;gW_Ly5QjXkh)qi1z*2e*g|hSL7{y z%FLrTaGUYx?VO-W4}WR+F9OvmwJ@&`FE~IzqacL(fY2-D(`zDMrrBC1K0J023lhAJ zjly0;`c>cBt->pmn%46(Y{9P~QTl(`WdRHocoJr>Zkz5WcuVXwG5PRkG8mmIn#fpW zeyI!pb$Q`Gt}+JplXS36}&z zU65#qn6MjoY9>cwZT}U^Y7o@rBf(dJOS{V&XC`jnK|bRMPgkAckzHO-+734-adUge z`BtFX7%Sx>YYQXgc+ZP>!#e{SDOKE_>RR1S+$)Qos@0zdZblTJP1Lx0zdkTwrhzfl zFZb#k7W~XprhiR7*@-A{RiR>kp;o*mwW}g*R{?32iQ(kh)iNrbkKy_2EWzJzqvELO zs2gmk-YwG2K{%iGH|1$w-_RHD$^0!TxM3sSZ1H=bPQ3B3l^HwTai0FSrQ8osqz5{c zZJKlU(DP}v?FYv`p0OM6zp;7*Nejuc6ct^ttsL}nuSzg;pbGh`ygE^ZxoERFa##zX z!if#U_y?dUt1|d8KUB#bqyP7;p4JEP4}hg6kVss%b1r?I2&UhG{jKF$d26tfNJrME z&nsL+PZeBwtWJ{8sG6|NAC}QTXvDLa_PF*36<=J<;r`{Nu%iZ+>zBAWb_p$yo8Kgv zuJP)t?|iMS`}#v8q`2>5$42NMqitRF z4>djN>B<_k3Flj_uN96c;^mohB*%(X?BTw)VByScTb^vV1e^+U{5cE%Vg%PMI<4vy z(V1UbUdwI>CW(0-mC#$fY3%T5d9Q}HdW9q`FY{gHk4B*>Ttu zu&hgpLZqDEaIRDEhpDp;wdjzN=m8CoyU0I4xub7-=fo!Fi)^GNr&lZ@jMWe|4F*G1ZDVYh9GfX#Yr%-AzNxqEd>JL8D` zq2};p))5lOG7HYLjJEa@px0)BVzNQj-hjLiZTYrjCC`Xaa=gher)ay|tqWZ=r9|;> zbsD$I%X6O&g{^pxrBMUp%svD{_V+E>G!2+yWF!I3zvq@IP+6TLfT}p>mc{-S2 zfn9Wk0Fpg8ItbsPcU>8 z;!H2>4cy5=vwNBRqq^-{Qw%#U#8C7XloyMx&4dSyRFAMfJfO6G`T+sw9|nqIx9S$8 zpMF;Qo260sr`aeR_+r@STX}vg0dj`a$V+L-+>86|aU}c)5Tcxhb?Ok@J*<=2_8Y(P zyW0%26FEJe#qNY-rsW9SFdT}R}q1zedC&D(8rUGR;`g;%3>gsAt{sdS6A(a`!tsevc zWhrd)b?`&%k~OL|@|?g&>K`D+8xEUkL>VOkg?lFws5(l+rK42GR7hBi`3rg&pLj9) z6k;!$T&(|Y_?obab;iJGqtVndqDDl&uQpBB&=fL1&v`DjrmopoES1osJACDSHv5|H zsP*%`uXRD?M9dWNLH51G%tMXG8W+3B8RXgd0~3X{C2?xccm1mLR&WV{k3fEZvFE83 zQ4s&PK=M>xaztx|vx8xS=e*%G?F*b$48m^xT8LdnF#P*pm1@g#ui36s*4!q&e}Gtf zW3qW04b_Sx${Yt|VcFh{^b{LB6xt~=q}99hIcJtDIjor74x(v4=$*f%{UDE8kg&kUHAAfL^P7 zvr)OoX7nt^s@o*w17-?wtf3y6@a)4;^^o&O-b=e(+GHS+7G4t$0WI*Lb43AvzQ1i60&yKmyA`tX`N_+d@i}jI z+&_W{`ZhQf2%IdJsB$+CenjB#s#fwA8L2y`=Lln6ce|!w|HACPx_`(kCI^UK3_nhz z3Tnkmo5+}mZbJ2mPC6PETwTJi92V5od?xv>HSaQ}_w-37V-FnmIB!kU!Or9`r{}1h z>r0;ZPK&e*onLguYw{{0N8$+-2Hsme&L&os-)`56+^gqc&~>p8%5q?$K{xX)Xtp*b zn_hW=R)731sOWyVb3SjhGS`x?v__J?3yjxWqRU!J%zL$Y$iIcA+^P@OoKY(3%d34| zXl5Nin{sFa@$e(#PUID$@cm!7NN8=vHf&I?EiA+HN-KN9#_JOnF+ks}j>ME$j9#m+ zvmJ;^u7n5gF4w?wWg%|fLf%n>;nhV{zCEwD{RI}f9y@V0>3_aTGv-HBut`{m!z0Xp{C&)&oSvM?B z-nc$+$?G-0o?~9zHL9PFF82e=j0b#i3Ysks81Ip=w+p7{x#Sl9m>mvBr>%jd@~nxWWXsWdP0)Al!NBw4D}Nc9>dPdr zv1H4Z_9mWK1Kb1diAnKtv_VSWS6lae{xf8Pfh(U%JtfCwV^GEqsUxFX4RzC7TD^3=nhnAVojQa z^Bl_XCy~D&V;9%ZDJW;`Z%P1$=(TIpS&x0~ps zy{sesyj6dx`5&MrxYcMnVwz_9`im)I!mvMK-gN&F4Y^(qc!acm$+bBnIVT8qQG7(8 zGQO)+Y97f|`NnLP2K$&8{@mH|j_k}VEPS~OVrQq;a;~7vbvNiNe6Xn0Y>S~&<5+Ah zdFTH0_Z59{W^d$3N#aM$3L9ZiAZpaUZ=%AwnuxKBtCw*Her`UQ^~j~-8+3L*)L32F znDOkKs~cai%quKlUoHFY9g-sV5BaVq-pofwnZ>^aGqqJS22kE-Pnz(9Vxy z8*)sbfDmsGD#T@eCSr=wh>K44p^bWl? z8tm)+FWQn%3tBOq2{3Ksv1v{0#E+<;Yb=!XwRH{&j*CYQ(%+O8pY&p2w2Hf*+ACK~ zf9}N`wrdf#?<^YSw+pIK!`A@;%MoE^2ScynX3(6sQ(VwL7VZ4em}%x2t>`o=gL#UF zgd2Sp5813Nx+=W#TRJbaxV8tgf^rzZ5rnTUd19UbWG0>suD zQyk0q)lYUn8BpmI@>3yYeC{!wEtp!I(mhfv*57voGSO^lWHIHYuw3v9`bPhKtmefG z>h*F0z+dY5MgP9ObFRM2U^a5IYp*#y_EH_p|2PslNs<Uj)@3-^<&|oQSs%hD+|!fR4&~USuql}y(=OmA8qmBU-!5jnkb@tjr;pmh5Z!P9)yy`Kx2oaNT?>l+7M>%X#U63 z{qy%T0@u-vkaDY(#mEvhMdOL1=c1G+2b~HvpKaG@sg=XKFuQ`gDp$HfNruAzh)+@8 zmg*j;Sqi0nncuvNZ-s9ck-OD1gP1wPljU4{Z+G=Xkh7~%ExcpL$8>*UFD?kf_9a>) zWpRFFlv|hhBE{F+ae#HQ(qjFM4BKCl<5`X>q$$rvooZMlyua6RI->52T9~pb5j}GF!BM$Rdq_2~( z?8_8qGiyp?Yh7ECm3+&Z*ViOAZKjKRLEYpHms!vm&u*{p4_Rm5Z}=K^H`TjxKlliWfacc|0(6 zH_-^PH5T;xM?(|BeP4?8u7&hQPjoRK%w*;6F~zw+xqiesD?~f%?W{93BX;2C(-jKk@3DGmOfOhEtHJ)-UW;=5qRt*9~|ea%1SQI=F%xVD2W1-T-fC&|OA)3*ijC;< z^+ic52ogc{TqByZ>lLT%*22RJNPWB5JwsYs$wvF39+RG+p6IYq)K`M%1fVSP%Tl+oKL?q0LM^C%Ve4Kuo*Rv&Zl6(A>YC+(MdexdR@)W(Ie~QLQD^{NXoQnSEbFK zM2n#Z0K?blPhIFCkdS$Oq3&kJ1G1e%eRvzzTAFSLa@+nNXEf;wX zG|w_0+wU*wZ>HMkCU9=8U^-szM6IFAz(~@c%cWu}0_Yw_GCv_*daiEIie;bavNYi#ec zCcjj>yetFpU-b%Du1=S=u~Z0lbE)jjtF0Cp`5hhRyZ9^Fgxk}qfPqMciqGz6AOMpY>J;@bMY&r8@cswy% zL2Nkn?z*MQvShemQx=t9ep4syc4)Iqi{;yByJV2%b+0m{Muq!IC89(%sj@L);_E;+ zg@Ol9EiAiQdK>aiO}K1Asuxk6B8R}^TRpv;t9*C+>U@LJ5?m%?ablNVAk6{&*{RDo z0J`8uo799j*^Qfy5Ue)lhMi)=%NH4=^U^L}jJ|n%;ib|`F1wWtNV@$*Ja7$ER-BVQ z8bVbhcH;kPe#5Vz+(6<=~tPxJAYX)S4(D^3}ILz!70x8jrc(q z%fuC@w|?M>-F(ukRvn%37u}?Z?x6Z9{n876LG;0VqSdR~C3~cB!x~bw?$(aYbKgjq z>+Nk!Jep#%s15lI>=HC;wcG1oA3`KQr|8Bw2O=LB)V}gX!Z*KKP(B%RY0k}%3SU~! zU!w`;sPn2S5SuCBAGTc>qEq4z4>a}uZu~Ia*t;%>2*HJdGSv6vbyQ4eW58^(hNRi$ z1L~45al}!+jNx>YLSeRl&bADWkOiACaWxan;rn1eJ@OvpjIdvUWxif4Fa_3<@fB@7 z$qq{ZGXc~L2|yyC^y-Vve*ln@6&8`e6gh_D=ro6%-1g;LNr9t}+>GM1y>W@}C?~>y zOFg;7h((L*v zyOSg!&A5c5eE$7jn;A#AM-PI3NXHt&eTGd;en?rHSf5D?KNrjia!!waSzGkxE$>I{ zS-CGtAEZfMZ%|av@QikpO0&6h2+&;8HPZ5VXNm=NsD5jSpYxUz!yX6rLB!^Bv=dlN zjAs#e3XnWgYYMkaCw3h1MRj>Hg=wBQukGPkIB%qLfeA zfn8MN!7;K!Q#xN>q^|n8V$!qUgmODo~T9tt!LA8 zX^vyM%UD5F`>-7}wODY~PS>B0JxQApF*$Uuf6Z|2)K*||D(}g=A}lcSO6?X>&co0s zYwbqmZwVR5xglyWJ#Mz~Fn07T?-VPXB)l0PsuGu9Iye5RN8#8>HZ+gJwe+=JzkY3T{;-Oy za=qpYlf{DPq$3Tfta`WsKXi6&WF5+{R(P}ylEn`XV`nZ2=}z*e6_rWoPw5GDM-zIn zHh`8c^`r@>b3#>-m%pD=1`DI>AeBCOhw-r_;?ai^Oz<{s^68UP6peC0-k`gY==j_5 zuL2&CpGAuELydWJ|4?Cvyc!D5qSz}QN$2sbMyKKs^=#{kNc?UgBbu0xRBZ)OFi|im zV#0^V-2C0+tQja~al4_t#*l)FsINpeeX04qk8HMO8+Cn%(9QQCq9u|e97pvDp+Nz< z^E}O;@9G{NI%m*r8J&6Uc!o(N^ok>bNHsp}en&npM0@_bB8%-8iud~{LfU~fX&xlY zM9p#v#YxA6te$+ylefrVO@$cw)lXDzM;<-aN!3$2J(o@^pmeyN8pkx~L0;1@U;TvH z%cKbsWQ>-o%#~K^r6yF9LjyU$1I2rS#|viO45?f%8Pw9gMljE~QrVeZ2)Ak1qj~$L zMN_I84chZ0Mv)xsf1aN?h^>a~xX5FRi2fN(G3A2Xj|rN$iIfbMhkoCKMDIj2xuj7~ zE~Uk{2h&aK<+e3I>49vc^xO1P0bweA*Io$){NBfvpJ_oJJxH7TtFmbU9v7>M6WQg( z&KYhYIMFJEcMcR~CJ=knfC`>(@>O`9D&A1h5~pA+E{t!SR{jqFrLOrQ+ssAU;HPyC zF&;IcusjQy-Jq)RO)Q@4eVH&lFEni;;T+iMlAkmD+xcwIMI$9Girr#?ct^Ry>UYe0 z^bxGo?s}Y+c6uvIt=o{|&KsMVYE}i1lE=)4-YS?k{k8T?F&|g;2WW=1sXy~UVD^z&1;PISHldOun|qH9d8ney)Y(oqTh5OvATALz7|n8cbzB&eAQNUlL}u0ddUI&`iiMX#5;~nd_y^!q<|xdd&J@>@d28n)shlu$ z-GJn)TiZ}6?qo6uymc$X__NGmZaP`}{_*;x-V#!HeRZsR9zGLYqnA9pGA|BaNHTe1 zH)be}0vjJI>z}>odK){1yWB{V2jnUyg1xDJz>+7^QN+d%1M<795j>2kSSu5Z&C93PXtO&Bme}efb0N*k1TD*+`8sM zi6nQYitxjSpQr=MC!4 zVdTa%mppxgS&Ms0LZ8Pm0F2mD$~!MH#8eqof_o0sL~13D=!WOo=yN)05(QyLIa1;_ zahuiHX%QXxy1dnf&Ze*Ny5q1-+*G6I@GpRHl}&GgTLe;m14glM6A7uSs|q$@rAlzP zVafCO^oqZSc@+Mzog}PGq#qf8wsHR%ltLL{pf0JH>R!#dqx>3AHWAFlmj@}(^^F>| zq*Dr_6(gq6WTWMsjOeY6A4wyKLkVTXudcl8U4KP-b)`*J^R|vBY%Ld5G-?PH#X6-e zzHwV7p2Mo>t&K{12g}oA;=Z-J=DP_ZzjUkhhAN3rtHmdhAUioN3zRJH1~ydti~u(F2_6N(C4aDrem2Hda9+R!LP~c z1bbT7N{~cE)$CIQe~sgaZkxX42+&}i@$>BZy*ud%UbFVL-Ztjh8v0PX5SG39tk#Q^Haik8i;71z$P)4x4&xmWzAT9Eo*@HFzlrq#eM}Np@ z>HuIZO_nGjBIqyGH!2)9?JGlE1OD~a)1IL|f=3#O3yjx7dg16scO1!EGDs8*#sB=F zt2>ST5o(eaF*SJBXy%!~YKYZto}^>Rnj zLBdIAVSHw<`J>H1$aSBj+3!}WsA!!U3Zh-wDqF`myPGOz6p=|_PPT}YK5>w!lOXPR7}B6^scIzD{lBriF&H$`Dt(r|596?q!rY1VnaE>L?G;4=n=t_M{5F|e#Wh_%|!;?nBP6b;|X6EL!qv{-ZeqBCA3B^PVsdBvdqMxv) zcqxPekt*!tSy7%8I2MqJXvGqxEq4rIeasoDCF{3AOwdXm*2fjwOb~)CRcaF-pZ{oliUS{~-Z)VJ;JM~&@0xAYFCu#I@M>jRp2Fav)H zCoEd}(K!{mojSH~5!>p zdjBUCnC=zzO`dk@3`uXc@)=NE`8wBN$>irG$_sY|tva+<(neKx&(`Q{_L6A9M5@i7 zI&)8PL(ee;N{*M7*BHqX&YWl*V`ccnecnIqVp#8IlY~QU-n;r*0j6r{J;(qU`{F>6 z&x9>AM}K84%l{U3KHjJPFsvV`o}fVaskiQEWliz3@3yUF+nXP!SFQT0)raRH`kixFDoINW-tdaQz0YsLH)rqT|>^Dt5jXWVu6!`sLN z#lFX}JB^M>+NP=&tqnHgddr^_-&SL9HJtM^@~8Htg(xt7Jjk;S$(wu3C2*wo-ptJG z@9*m_fB&L0&qdB49(x$F79+2FVcZ%iHJNJH3Q!H-#1Gv6Q^=i01*vz>_GKBJ@yMR9j{zIYoB^ANEUd{KQGoYJTlTWxpuCBMw=+YwG!g(9W14 z(F;F-h(eL2r7*j%jWG<1TM4{OuP@hI1=Wq>Bgt^lo;!--`9JqgVi0acUvpib7(GH$ zHd=Pe3|LH}?lRy4u@Orb7DU&3nqd7okX<6a6pFh5iyFk#?}#3YnobG(-+Y>|jqRA4!4p z%$9fu_o3X9pJe(v2Dc5>{glX>HY*7O^+(tsrVRI1AummSu^!xjuB{xRBb(Y~X2ySO z$8eFBoCh{FZ4g9nPs8bk`VGCmRxW{RFe>l+Q4^WFdObxsouX-62e!@!<2C)(;*6+ zz`aJ6$3%s_Qr0JojOiZV>a7lvZcQ5RcY7m<#xal79={K#m^~jqJ`|33=JIw9 zN*MOpHPU#eB>oY_Y&(rVlV@rXRI|O`k4Qazc)jPVXb%_ap~<6j0b<(N@!-5m*GJIT z*ZWruIcXEeZ;9i_NdLLo)GXJUoky$qY0rI%g8pq*;U)%XaK;Z$RIw> z(H*CMfGOqw-B2N2JYnd4nhq4xP9ZWRpZrZe_F`5eH^58jMT2Udc6GRp#L6`W>qQpx zuR6TB=H2r}0#RMrW>Di{NHiO2Y(}Qbe`8Dk59AY=^t_TVeo93&_}TY#62?aBg@+N9 zbqdk*>xid)g8ZwKwIZ}xZZT}Z8B9g};2;elYeo^%7av9Q|4#t^BmvuO_WoJf{#x2`i* zbZKIO7GNx*QG96sOw(zu-oRx_1r6(j!7j= zd>xSFs$@m;&@}gKCZ;6>l7t$#7r}2auV2MWQC48xkIVaHKu>2*VnI}mZ zcic>HrMVz?HOEsAhN7F6r(9i0>n#zl{{RU_qkC-ka$5bKDbvZ1Ez1o2#1GP}_`WFN z)z;c2*&Hw;s*Zz<@m)A-G^Z6&M#mj&M56g(b60Y_&Rxzc7(6z9p1`Ez~N>DHW5zM_+AH%jJ&-)Ei& zj7uI_a5+EWU5%%Pn%3@l-%qw^r)>F6wUs`Z>G{>VbpHUDk8f9QLyWV9(#ZpXk&(gA zdhG6eCn$l$HsKos0d{~$?s4s0)v2ZLMcnJd*KggNY*FwU{{W^<^>5^9d=l$#sY?{! zArcjA_U&G5cPY=%=lWIK+h?`9k~tR#IH|+Sm)(1pTi;4c zW6bqN#l`b_Gk-c=UN-YJ91cS_AB}qOe)LZ+qW2_Q*QRUzdh+d%BTV6ngOEN@KcyzC zBz|*C69!2@Wns{Q1$M_|d5_|DGbEF4QGXNbQ8!w5A1DbKSTdXu(wjmH5(X^AiODA< z)$KOTWKTLwFeHGdrA(?Q?r9oMOG2u#t!eCIdwE_o3nXd01sDVlqaRLx3Q?-0>?zT4 zS1Cm%JM&s`#IT}8ft6gb$5MLIb<%niu+6Ks7FUG``G`5|QZ2E$7?1s8uQ#5({{TAXR`uYJh>Gk}z$y-Z8lPzw(4|Iesp{5Rn$3l5XSH&= zZoOs#ZH$&9_`4c>iL0W0qlB#^yP6AaPSLI983b|ar_@yqF7YQ}gV&{G(}}8anmDa* zM3TzsqIK!e`j1-gb`R5ru?&xK-mWw5XDW8s^8WxawB0*WnGso<%1F8NYy(sH zltXEv-K)r`@}|^Md1H~%s!u{-pE5apDL3k&2L+Lb{yR_6hBc)W+Cw6PGJ{fn3UgpG)C3c2JOcC#s*Zga? zDmJ@kpHfo2lukUd#1tjOX`ZEbXCIYzW5WA=Q9qNQ~+ODf5#q&iQ5d=pB1NV=uWCM-u(6P;NN^P^yjX1?v-?6>P19v^EjMg=u z?5I{$mutIV4AYGJiO_P=DDw%;|>jM&HC$LK3LVv^I8 z?uJoQessM*q;*njOW@xSJ*-U9=`6!~6O$1<K}e$g;+=M!}K5D!}kXdGEtLc27Qi3hr#|Q1+p_DErDjoFBy3sY4NR~{KB%fL3Vce(IF)5Obr*{{{V(`$pp88ZB9UP1{dcW9B@RjM2j3~Fsh>f4`EKEFrt7KibFZCh`;hr^`H9p{-TeFzw%G@ zpZfRyqJ}v9Nbp$40+un-prYlt=})&EDGa{?^r`J4-;zjGQI=!aPzzUMLe4$P>3~b2 z1KOx+ULlU!>N{IIRuHP@Ns0WvrkaO6<0bCfaVoQ?ILe!})7j`cPD#hgN9jc9k#Od2$lLIVRFX~KdO?BjiouTMtoJlhl2`6B@mfolme;#Exn|^=R*z2S zrN_5$-SIZtltv`0s*@ZfnZO5UZs%WioX17+O_wfue z{{VXxv!+gxOFSyU_P)|V@89*!e8lBb4OPJRC|7POq#5UPyc6ath_J(g7+{L*HQx=# zc+DV;D|(K#*Gm~q@1r{2DsonoRF6KJOSdGnGctZPa{EWSvvM~QJ?qegWi6y~PE;QB zt>|IBh9Bl)^?S;jb_;$rx~T2B$0*ZpNR73&jy4;HPh}nJLr#h$Z!$x`?rR#AN{a2F z+;}?M$pYd<%&a~^=s5zX@Vx&3XuI5_2%~5o!oGhMDau-%HcBREi&HxTgNE{vocAyEso0*(l#zb*2Lg&;UTxTP_VOI@KqU_D-VroN2swKX&hgPy#t0;4YRh|mCEBQ2zHD^&t*Agn<75*D+a6 z9NMF5b<}ibMT@7~p$SaLI`+D1!q<<=p- znj*^E9IjO49(x+*#na>+$7$y{G;#+y*J+Y0abDbHVmL~Ud$uwBtC3r$5@h6OwH%BU zL~f%DjGjBzkA(a;bK)d>gC~({pD2?VF5ZBDTCP?ubcn3ntc#9tIj+VZ07nr7Y7!Iw z09?oT*Gy#?s~j~UDa!U2M!X;{4h>qk(k*m52fVtzWr{xHJeECx9mzZaNy#xr6N#^V z;%j{$N_)$~TW=A@zj{XOqP&w^)u+|2y!(u-OO=igdJj*p_mAK!2g;7P#!VIPZ!R@! z=F~4RS=;wfFDg0VSJ-;@HDb~qB^v5-Jixyw?7o7romTWZsnJN+@ZN_c_aQ>=mQGxr zfPaUrTJZOYW3zxkEJ2l@bgZWgCb4}hwEepaEp>0m}ni(eh zh%vcWusIz~bQTdCI2(NPNEwTdIr3CjbZJ5=$mE@A&E1jBrJ_isH@dGN5;42i1HE(F zrn7miY6iniGmChJ>E(cBg7!b)KN{LJ9QHZtVk1v-<<;G@Wnb+GIx!aVay>JSmCxKv z@XxZ(#23;m45CGK=sS#&^AdO%rz}4zyJSY^YIy8;`Ck*NdYK z$i=yF+UntV3)vI7WQh6g*n^WLY}N2@uiW|GbA8!j8+CPscLWU66;W% zwYl|3h%+_A$P06vll=`sR2dj!9OPFzlic*(D7DGZk5s?9)-EENWkKY}<)Q(`+~=>< z)bQKxaaOSshRo-d0+jSb80t80D~#l>nhF8y;|{Jvy9yds38|*ifSfx$3%JmuaKerJbC6 zBVI{7mOaTm&0DM+xzv(M$XwD;F=(Iyl8_Fhpar6VoZrM>`4Baa`uF~#kBFc0AZtJM z@BKw5u;cL~!Xh0h@QmW1qV2NeiqO)nl*lBP{Zf3*@9jynw;GJ3?G<;}u{wDcF|I~Q z+~fIGJwogKA={!cBzsj(5uAOTou5b1n_=oyq}6ywnh!}YwI22(y4xvrBYAF_9crvn zw6?p{{NUstabBFLO0FG7_I^i=M-fj8jFgmlqV~S%qxQSa8gm1oXj_nt>FNz^P7_kH z7Yw<;>^|xAuRfJaRQ77=o~?X4ogZyJR-b2Y#r{Ngx3;&V(@O#pE}4! z)0I7FsZCXrk1ng^&2ZIfVPP7IqvU!ed#{zzy{KO;j8V$0TH)LX(>;e$eY;mRaU8K2 z{LJoikWX5|R1$ZR(79KYNT_pO-`7I+qkk!Td0`$xtCo-_%ek}Ef!3j0W_WzUzzlJk zBTG#c(JpE-)}l>%kY8%Imy)u{Czl(yDh38U&23rf_P4Pws2m&)Yp30Q?KgMZW0}&F z7qqOoSMx1b;A4g%4*-Ks)JNH_4YIJ?j>428_2^Yu#!X!*U$0;I0%`ZMY4=fGL60&c zxpUC_R1ikew$at7(wLsdsJw9V+$3%wY45d)5@^DC~7w=LvVFNf}~$b*{LC6B27CSHg1u1mvrW*tRXleBjA^sjY7jVdaXPX=Dn zRlTXJa4PnxW#ny<4(OyAQ|at$iE4Kyna|ET)2;_5sR3Tn{q)P|?vX=n%Cev0Q`A*U z!?iQ-RF&9X$n5Uz?R8k8)Zl3uO8ov-H^`uLlN4TzU!@B9xu6((&)^ic&0o4y@ z9^$LshT93K%X4#1*Co+!7Uxg3iUu4u_h&LMp<~$l3f+UldVZyTvdc3FnRjfDfhV{< zhp_Kg?J2dXm$ImyjP5jjZpXwrc(}EbJ*14BJWc3*5231dRu$+;3KF}jpNXs z^^0oy9SJ*XL8|F8M}79|W=Q2HbFm$N+8F&u_}0Al)`?_pE~HqYA9%=dgX#6AruHS% zTby3C;TP24p7%&`=ER|;w8l3c`#nG1H6`zgEp3?=$(qs`PRO!l*!$N) zjT9Eb>5O56e5iAhYpOI}_c-T?Uk@&)S2n#2wx}Z0Wwv|(42_Zh0N1WM>tDILm&}u4 zJpdWN`d32bM+;RXbd9V|y`%iAmR&+=LBK)z)h<`Dn^Rk}D^%4L{8cU0$|R_*!j#IK zE-{bJpb{+lqay_Qi5rhfgp-ld(^T~_uV#$OL^<;EcI0#wJ;nsh@{SK`&F*a!)KWM6 zC97NMwzrZ=6mrcXVRw#$2aNtTmuoNS9l4x`rFLRfm*9 z5F7j53H++x!*L|Gvm{0&D8QY-l4zZ|Rm{|-?H!K$7mE>e@|=P9a7Jp(G2De2Xn!OA zHP4~euVaqab$i>`OfqRt6mjIbnFtbn4hCzf)h%XwDHmjJjmxM9j(H$|N*$WA66TA$ z*zre-Sm!cH3j_QrtHO^#U0%0tmX~28DC$P#U^pV7(stb57>K=*$IBJ<Q8FyKuF;8BA7)W!z_p}L97z9vDGJaaeO=AyNx>HJFP|HV!t*OvLpWh6YtPdlf~_E zW^JaH7^Zd2z0)q}T=p%BzEyKCX(qQj1enZ2V{A+J6!-T(&{sEQsB2di&26Yh1XCaR zZv$i$^&aQ9=j}t#m8jnm_^R112AU@*gyMNscWw5^e}Bu?v-MvL>9=Z3YS!|9dNaZL zgH;JZZYjoXSrA`dY1(os_-9+Wn)36B0O3IH&yX?Rv4y>_hiv7zl21L!#_VIvJ(Lem zTG`cBv`%G**7IShYBA`xDGWz+%rh8nr1z1(psqgp_FJooE@y4d=bvvtJ*sTmyEVE> z$ojH_NC>WNp}T?Lodd?>1R9c+fi{mi@n3-NrqkwKFpH2zL~X+Z=sj!EY8^PUcw#C+ zpD;W!>6(C!{?UU(-#ij^85{3Nr1cm-QC&BUyf>wIF567g*0}K2kJwBBZ zqb<>;EM+-p-mCF5>>(0NwO>rVUo<2!%XKy~v0I^$+wc|5DpelH?~W#l)yWcg%!qXR zdvzZ??T|f(Q&&%d{9)oN)w+{MmJ}c=dCU+Z`e2;?mC=W$>3hv;dHA~U#Cx2{MC5EV zn~UkAf+*fu9fG1SIVZ5Mdhl0(qwwGtcdV1&V+*i+v1hJ5PrY<=Rg>35aZBoym%Hyj z!}|0scniWQ;VlcukgHx?H<@tD&>V21*bnPnMQD-3DLFf{gz-m^rZ;XTIhWCCKH`krmUCk zGU;+VBKeR>_tVh&ikj{-8yMdQigtxqw+HtGT&c+_vAw!6kX+i!8#%d*G3pBdN8wpl z7I9>X$Qlg`;;0I)_h}^Q#)=ovp00z205bh*-(v@~_b2QZj;)dNbsd zB&6cjr3iHyuWkPTdoy7PTn7IDIjUN2k$tMkB#@IDlhAY(^SISPRpyIG`y8th=+We< z8?n+~NRxSDLdr)*Ijt7(P1HzcX*|~h?jDA-!&bs`+HEp(8kK6o^rLUg%4V1;kc12V z_cUB-vV*caY1M~NbBgxV@bFhw{12Mttv6?X#Lcd&wQnku9#L;lCT^`5C_6KU)CM6xQBTzdE!*lpopi_I zGM2`4Z;mPu)v21f-S|7iCL|hc?mknx-I(I`Y1DjI zZ)0qzh-bTXz==;oj^@0&PZ8=Ggb~`t6ifF^%06N}2c<%dw=bIe8=AL>E^aOsIapoZ zXZLWA^-z8t&1`rp!ij7a^6E1z&VEEaSpNW?rm>xP$3$%!4XenZX`;zwqf*rz$#XB= z-g!UZD_h1yMS6(L9$mjS2Mg>6rA0YPtJKnQjWwb&rMp(o%rH9EOw$${M^5$C7+B?$ z+bm6bWIb@Bpse!Mt;~5*YtGG756u@HjZrs|Ve0ea%P`u0wa<87MjGX=4@m*4jIZvqI?zksN6<0K0*-4XuJ*yZJ*zHPa@9p+ zVdq>PYtHWcV9`wR=b^!5sS_0P&QQctcYa=fNp-Z%Qzm5I+oB|t2?bUG`8#KS~=YD9-mR@YqQX-)_W%T zeAuz`CsMxS*Xdl6tl;d!0qwie|Q5I*ohfZv5zv(%q! zW#(AcwO7=xcb}FR#v?({dYZ_U^T0K#jf`BI8q8#GVNA{|O^9dLri0C5Sec^~08-+C zg%ki$iU26Z02EPRK5+Pr=g;vIRTDIfM5>Vn&)wQE2e)6=z1LLmeuJ%Et>ylnvQ9CU z3|D?SZo%jAtBl-w5aV;1_(|b-bgvJtwIb&4Qcb|0;EO*fA6}>H?OpUx#{?-1C=xi- zDk_Wx1E3V7+?dj^w797#N?cF@MkoO(aX<-6iUcJXpaiAG02E??3S3YEQsRILF+k3D z;s-ti)rb8t{{Y#A9}sx(Ca!sc zVEa%ZAn;B({5yUX@wnOZ;w8*|1{x_rBR|A?d>W0nm$wpR?Ix@1GR!5C&a$5UTNN;N9`t=RJ`RE-tSYnv9U z9A-Hj29a10y}ugIONpQ=(ZrHt?(QUG{A*=`uTEa>sN{I+6=QijGg|IABpc?)xpp`} z$L;~a{P?W@05(b7PcsiRt_eYbU9rOAd41V(oRP+1BU1Gt%o19~-#?uraz17!@v2x% zOWvWc_J#p`Gzh$?RQ#w7AyxDg{-IItQU3s6F|5*p+JgSRLnSGKFa;y+9RL)*pt{Y2 z&9Cs%NX)tW#{krqEDse=4NfK>T&p%`NKQJ^x|7gWc7#e&t3KwJhb4tC)ny<8d-SR{ zEB=%dx$b&b9wr{qXm(-Ji?p4Od(y6<`x<`n0LBL*yvo@x?bTivka5)>zJ7UIz3%?w z*HEWFhNOP;&~-lz>H3w)xtTXH^fkoCuWMGoD7};c(OpOQR$o=b!`+1R`J1c*dfh+k z(CqCjtn}qD#kB(RbK1Di_(`=K<@}d9&ne&6txT|WW$`<|QogBGZDYDJM=FwxtUW=l z4o?+og|`C;IKjeo{c3)pQ=txpMY=lcl!$%pqbD5K19#$0QbNpLES^8r^cB544Nk1e zQIhCv>t0Mj#^uNvE1qkOxYTav`}3Slhf%`bX#_oIMPsvI)S_9Z}9rp1zJ>LCpwPr?>!1w zj##MDhP~d#rkSYRiHtGD=0X1eEfL04kM{onKAme$9Xfj(h@_fjj@?@xTw?>C=hyMB zO4ztg7Kcx~(h#cW%~MKg^zNw0CF&Er1*iS*hi)PrV+B>vTg`C zAf5>yf#=&5-IKgO6z0I)d{N#0Iyc(9f;Ndahjcak7`$AvN{hATiIH% zyS!krN6F)p><6u7z+QO~21w7!D;kly)twZi*TS_uYr}pbg7G9}i^_1P-(rq~{P(Ud z?^1sUC9E<+9j@Z@2|4U~R|=-8OZR&nlA45ndAYl)`BPrBl1KK7iH;Rm@fh_YxqtXg zpkur2Nt4O@+}CU=xT_iTk}{Rt@-pKdpPxTLk6Oi#$0&uFY}6Jc zo=+t8tL0{V$m;DhtvT){XR(As4iplgvrCr8B^LSa zb5X~(Kus&j^Ko4ENc0Fe?z;5#F@sFm%m(w-*uZ6++T|(hQm}aI=P{j)Q7p z)gpX!HKnI*6023lDHv3$+OgZqb&FHaYRH=BHpFMOI~jAgp~UKwk$opUTC3{k7fgLk zM4qQ*II9)jV%_Sgdd;(C6kdh5b*k_^WhTwylRX1KI+m5C4N3=w>I2q19m`R86H!vq z*h+GejsdSH6*`cW-l+6w!VWh^M~OA)^+`7*LM3il0sF1o{S8gy9}YI3J5LIe3-iZn z+YeHdw%pFFCYFxq;;vv;W9v_IgM(e^oG#r;Ee0yd#xYu5nb{PXF;*Etr*T;S*(nFn A`v3p{ literal 0 HcmV?d00001 diff --git a/modules/meissonic/__init__.py b/modules/meissonic/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/modules/meissonic/pipeline.py b/modules/meissonic/pipeline.py new file mode 100644 index 000000000..512eab742 --- /dev/null +++ b/modules/meissonic/pipeline.py @@ -0,0 +1,373 @@ +# Copyright 2024 The HuggingFace Team and The MeissonFlow Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +from transformers import CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers.image_processor import VaeImageProcessor +from diffusers.models import VQModel + +from .scheduler import Scheduler +from diffusers.utils import replace_example_docstring +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput + +from .transformer import Transformer2DModel + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> image = pipe(prompt).images[0] + ``` +""" + + +def _prepare_latent_image_ids(batch_size, height, width, device, dtype): + latent_image_ids = torch.zeros(height // 2, width // 2, 3) + latent_image_ids[..., 1] = latent_image_ids[..., 1] + torch.arange(height // 2)[:, None] + latent_image_ids[..., 2] = latent_image_ids[..., 2] + torch.arange(width // 2)[None, :] + + latent_image_id_height, latent_image_id_width, latent_image_id_channels = latent_image_ids.shape + + latent_image_ids = latent_image_ids.reshape( + latent_image_id_height * latent_image_id_width, latent_image_id_channels + ) + + return latent_image_ids.to(device=device, dtype=dtype) + + +class Pipeline(DiffusionPipeline): + image_processor: VaeImageProcessor + vqvae: VQModel + tokenizer: CLIPTokenizer + text_encoder: CLIPTextModelWithProjection + transformer: Transformer2DModel + scheduler: Scheduler + # tokenizer_t5: T5Tokenizer + # text_encoder_t5: T5ForConditionalGeneration + + model_cpu_offload_seq = "text_encoder->transformer->vqvae" + + def __init__( + self, + vqvae: VQModel, + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModelWithProjection, + transformer: Transformer2DModel, + scheduler: Scheduler, + # tokenizer_t5: T5Tokenizer, + # text_encoder_t5: T5ForConditionalGeneration, + ): + super().__init__() + + self.register_modules( + vqvae=vqvae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + # tokenizer_t5=tokenizer_t5, + # text_encoder_t5=text_encoder_t5, + ) + self.vae_scale_factor = 2 ** (len(self.vqvae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_normalize=False) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[List[str], str]] = None, + height: Optional[int] = 1024, + width: Optional[int] = 1024, + num_inference_steps: int = 48, + guidance_scale: float = 9.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[torch.Generator] = None, + latents: Optional[torch.IntTensor] = None, + prompt_embeds: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + negative_encoder_hidden_states: Optional[torch.Tensor] = None, + output_type="pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + micro_conditioning_aesthetic_score: int = 6, + micro_conditioning_crop_coord: Tuple[int, int] = (0, 0), + temperature: Union[int, Tuple[int, int], List[int]] = (2, 0), + ): + """ + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.transformer.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 16): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 10.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.IntTensor`, *optional*): + Pre-generated tokens representing latent vectors in `self.vqvae`, to be used as inputs for image + gneration. If not provided, the starting latents will be completely masked. + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. A single vector from the + pooled and projected final hidden states. + encoder_hidden_states (`torch.Tensor`, *optional*): + Pre-generated penultimate hidden states from the text encoder providing additional text conditioning. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + negative_encoder_hidden_states (`torch.Tensor`, *optional*): + Analogous to `encoder_hidden_states` for the positive prompt. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + micro_conditioning_aesthetic_score (`int`, *optional*, defaults to 6): + The targeted aesthetic score according to the laion aesthetic classifier. See + https://laion.ai/blog/laion-aesthetics/ and the micro-conditioning section of + https://arxiv.org/abs/2307.01952. + micro_conditioning_crop_coord (`Tuple[int]`, *optional*, defaults to (0, 0)): + The targeted height, width crop coordinates. See the micro-conditioning section of + https://arxiv.org/abs/2307.01952. + temperature (`Union[int, Tuple[int, int], List[int]]`, *optional*, defaults to (2, 0)): + Configures the temperature scheduler on `self.scheduler` see `Scheduler#set_timesteps`. + + Examples: + + Returns: + [`~pipelines.pipeline_utils.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.pipeline_utils.ImagePipelineOutput`] is returned, otherwise a + `tuple` is returned where the first element is a list with the generated images. + """ + if (prompt_embeds is not None and encoder_hidden_states is None) or ( + prompt_embeds is None and encoder_hidden_states is not None + ): + raise ValueError("pass either both `prompt_embeds` and `encoder_hidden_states` or neither") + + if (negative_prompt_embeds is not None and negative_encoder_hidden_states is None) or ( + negative_prompt_embeds is None and negative_encoder_hidden_states is not None + ): + raise ValueError( + "pass either both `negatve_prompt_embeds` and `negative_encoder_hidden_states` or neither" + ) + + if (prompt is None and prompt_embeds is None) or (prompt is not None and prompt_embeds is not None): + raise ValueError("pass only one of `prompt` or `prompt_embeds`") + + if isinstance(prompt, str): + prompt = [prompt] + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + batch_size = batch_size * num_images_per_prompt + + if height is None: + height = self.transformer.config.sample_size * self.vae_scale_factor + + if width is None: + width = self.transformer.config.sample_size * self.vae_scale_factor + + if prompt_embeds is None: + input_ids = self.tokenizer( + prompt, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=77, #self.tokenizer.model_max_length, + ).input_ids.to(self._execution_device) + # input_ids_t5 = self.tokenizer_t5( + # prompt, + # return_tensors="pt", + # padding="max_length", + # truncation=True, + # max_length=512, + # ).input_ids.to(self._execution_device) + + + outputs = self.text_encoder(input_ids, return_dict=True, output_hidden_states=True) + # outputs_t5 = self.text_encoder_t5(input_ids_t5, decoder_input_ids = input_ids_t5 ,return_dict=True, output_hidden_states=True) + prompt_embeds = outputs.text_embeds + encoder_hidden_states = outputs.hidden_states[-2] + # encoder_hidden_states = outputs_t5.encoder_hidden_states[-2] + + prompt_embeds = prompt_embeds.repeat(num_images_per_prompt, 1) + encoder_hidden_states = encoder_hidden_states.repeat(num_images_per_prompt, 1, 1) + + if guidance_scale > 1.0: + if negative_prompt_embeds is None: + if negative_prompt is None: + negative_prompt = [""] * len(prompt) + + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + + input_ids = self.tokenizer( + negative_prompt, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=77, #self.tokenizer.model_max_length, + ).input_ids.to(self._execution_device) + # input_ids_t5 = self.tokenizer_t5( + # prompt, + # return_tensors="pt", + # padding="max_length", + # truncation=True, + # max_length=512, + # ).input_ids.to(self._execution_device) + + outputs = self.text_encoder(input_ids, return_dict=True, output_hidden_states=True) + # outputs_t5 = self.text_encoder_t5(input_ids_t5, decoder_input_ids = input_ids_t5 ,return_dict=True, output_hidden_states=True) + negative_prompt_embeds = outputs.text_embeds + negative_encoder_hidden_states = outputs.hidden_states[-2] + # negative_encoder_hidden_states = outputs_t5.encoder_hidden_states[-2] + + + + negative_prompt_embeds = negative_prompt_embeds.repeat(num_images_per_prompt, 1) + negative_encoder_hidden_states = negative_encoder_hidden_states.repeat(num_images_per_prompt, 1, 1) + + prompt_embeds = torch.concat([negative_prompt_embeds, prompt_embeds]) + encoder_hidden_states = torch.concat([negative_encoder_hidden_states, encoder_hidden_states]) + + # Note that the micro conditionings _do_ flip the order of width, height for the original size + # and the crop coordinates. This is how it was done in the original code base + micro_conds = torch.tensor( + [ + width, + height, + micro_conditioning_crop_coord[0], + micro_conditioning_crop_coord[1], + micro_conditioning_aesthetic_score, + ], + device=self._execution_device, + dtype=encoder_hidden_states.dtype, + ) + micro_conds = micro_conds.unsqueeze(0) + micro_conds = micro_conds.expand(2 * batch_size if guidance_scale > 1.0 else batch_size, -1) + + shape = (batch_size, height // self.vae_scale_factor, width // self.vae_scale_factor) + + if latents is None: + latents = torch.full( + shape, self.scheduler.config.mask_token_id, dtype=torch.long, device=self._execution_device + ) + + self.scheduler.set_timesteps(num_inference_steps, temperature, self._execution_device) + + num_warmup_steps = len(self.scheduler.timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, timestep in enumerate(self.scheduler.timesteps): + if guidance_scale > 1.0: + model_input = torch.cat([latents] * 2) + else: + model_input = latents + if height == 1024: #args.resolution == 1024: + img_ids = _prepare_latent_image_ids(model_input.shape[0], model_input.shape[-2],model_input.shape[-1],model_input.device,model_input.dtype) + else: + img_ids = _prepare_latent_image_ids(model_input.shape[0],2*model_input.shape[-2],2*model_input.shape[-1],model_input.device,model_input.dtype) + txt_ids = torch.zeros(encoder_hidden_states.shape[1],3).to(device = encoder_hidden_states.device, dtype = encoder_hidden_states.dtype) + model_output = self.transformer( + hidden_states = model_input, + micro_conds=micro_conds, + pooled_projections=prompt_embeds, + encoder_hidden_states=encoder_hidden_states, + img_ids = img_ids, + txt_ids = txt_ids, + timestep = torch.tensor([timestep], device=model_input.device, dtype=torch.long), + # guidance = 7, + # cross_attention_kwargs=cross_attention_kwargs, + ) + + if guidance_scale > 1.0: + uncond_logits, cond_logits = model_output.chunk(2) + model_output = uncond_logits + guidance_scale * (cond_logits - uncond_logits) + + latents = self.scheduler.step( + model_output=model_output, + timestep=timestep, + sample=latents, + generator=generator, + ).prev_sample + + if i == len(self.scheduler.timesteps) - 1 or ( + (i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0 + ): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, timestep, latents) + + if output_type == "latent": + output = latents + else: + needs_upcasting = self.vqvae.dtype == torch.float16 and self.vqvae.config.force_upcast + + if needs_upcasting: + self.vqvae.float() + + output = self.vqvae.decode( + latents, + force_not_quantize=True, + shape=( + batch_size, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + self.vqvae.config.latent_channels, + ), + ).sample.clip(0, 1) + output = self.image_processor.postprocess(output, output_type) + + if needs_upcasting: + self.vqvae.half() + + self.maybe_free_model_hooks() + + if not return_dict: + return (output,) + + return ImagePipelineOutput(output) \ No newline at end of file diff --git a/modules/meissonic/pipeline_img2img.py b/modules/meissonic/pipeline_img2img.py new file mode 100644 index 000000000..f26af123d --- /dev/null +++ b/modules/meissonic/pipeline_img2img.py @@ -0,0 +1,353 @@ +# Copyright 2024 The HuggingFace Team and The MeissonFlow Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +from transformers import CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers.image_processor import PipelineImageInput, VaeImageProcessor +from diffusers.models import UVit2DModel, VQModel +# from diffusers.schedulers import AmusedScheduler +from .scheduler import Scheduler +from diffusers.utils import replace_example_docstring +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput + +from .transformer import Transformer2DModel + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> image = pipe(prompt, input_image).images[0] + ``` +""" +def _prepare_latent_image_ids(batch_size, height, width, device, dtype): + latent_image_ids = torch.zeros(height // 2, width // 2, 3) + latent_image_ids[..., 1] = latent_image_ids[..., 1] + torch.arange(height // 2)[:, None] + latent_image_ids[..., 2] = latent_image_ids[..., 2] + torch.arange(width // 2)[None, :] + + latent_image_id_height, latent_image_id_width, latent_image_id_channels = latent_image_ids.shape + + latent_image_ids = latent_image_ids.reshape( + latent_image_id_height * latent_image_id_width, latent_image_id_channels + ) + # latent_image_ids = latent_image_ids.unsqueeze(0).repeat(batch_size, 1, 1) + + return latent_image_ids.to(device=device, dtype=dtype) + + +class Img2ImgPipeline(DiffusionPipeline): + image_processor: VaeImageProcessor + vqvae: VQModel + tokenizer: CLIPTokenizer + text_encoder: CLIPTextModelWithProjection + transformer: Transformer2DModel #UVit2DModel + scheduler: Scheduler + + model_cpu_offload_seq = "text_encoder->transformer->vqvae" + + # TODO - when calling self.vqvae.quantize, it uses self.vqvae.quantize.embedding.weight before + # the forward method of self.vqvae.quantize, so the hook doesn't get called to move the parameter + # off the meta device. There should be a way to fix this instead of just not offloading it + _exclude_from_cpu_offload = ["vqvae"] + + def __init__( + self, + vqvae: VQModel, + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModelWithProjection, + transformer: Transformer2DModel, #UVit2DModel, + scheduler: Scheduler, + ): + super().__init__() + + self.register_modules( + vqvae=vqvae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vqvae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_normalize=False) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[List[str], str]] = None, + image: PipelineImageInput = None, + strength: float = 0.5, + num_inference_steps: int = 12, + guidance_scale: float = 10.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[torch.Generator] = None, + prompt_embeds: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + negative_encoder_hidden_states: Optional[torch.Tensor] = None, + output_type="pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + micro_conditioning_aesthetic_score: int = 6, + micro_conditioning_crop_coord: Tuple[int, int] = (0, 0), + temperature: Union[int, Tuple[int, int], List[int]] = (2, 0), + ): + """ + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both + numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list + or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a + list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + strength (`float`, *optional*, defaults to 0.5): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 12): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 10.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. A single vector from the + pooled and projected final hidden states. + encoder_hidden_states (`torch.Tensor`, *optional*): + Pre-generated penultimate hidden states from the text encoder providing additional text conditioning. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + negative_encoder_hidden_states (`torch.Tensor`, *optional*): + Analogous to `encoder_hidden_states` for the positive prompt. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + micro_conditioning_aesthetic_score (`int`, *optional*, defaults to 6): + The targeted aesthetic score according to the laion aesthetic classifier. See + https://laion.ai/blog/laion-aesthetics/ and the micro-conditioning section of + https://arxiv.org/abs/2307.01952. + micro_conditioning_crop_coord (`Tuple[int]`, *optional*, defaults to (0, 0)): + The targeted height, width crop coordinates. See the micro-conditioning section of + https://arxiv.org/abs/2307.01952. + temperature (`Union[int, Tuple[int, int], List[int]]`, *optional*, defaults to (2, 0)): + Configures the temperature scheduler on `self.scheduler` see `AmusedScheduler#set_timesteps`. + + Examples: + + Returns: + [`~pipelines.pipeline_utils.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.pipeline_utils.ImagePipelineOutput`] is returned, otherwise a + `tuple` is returned where the first element is a list with the generated images. + """ + + if (prompt_embeds is not None and encoder_hidden_states is None) or ( + prompt_embeds is None and encoder_hidden_states is not None + ): + raise ValueError("pass either both `prompt_embeds` and `encoder_hidden_states` or neither") + + if (negative_prompt_embeds is not None and negative_encoder_hidden_states is None) or ( + negative_prompt_embeds is None and negative_encoder_hidden_states is not None + ): + raise ValueError( + "pass either both `negative_prompt_embeds` and `negative_encoder_hidden_states` or neither" + ) + + if (prompt is None and prompt_embeds is None) or (prompt is not None and prompt_embeds is not None): + raise ValueError("pass only one of `prompt` or `prompt_embeds`") + + if isinstance(prompt, str): + prompt = [prompt] + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + batch_size = batch_size * num_images_per_prompt + + if prompt_embeds is None: + input_ids = self.tokenizer( + prompt, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=77, #self.tokenizer.model_max_length, + ).input_ids.to(self._execution_device) + + outputs = self.text_encoder(input_ids, return_dict=True, output_hidden_states=True) + prompt_embeds = outputs.text_embeds + encoder_hidden_states = outputs.hidden_states[-2] + + prompt_embeds = prompt_embeds.repeat(num_images_per_prompt, 1) + encoder_hidden_states = encoder_hidden_states.repeat(num_images_per_prompt, 1, 1) + + if guidance_scale > 1.0: + if negative_prompt_embeds is None: + if negative_prompt is None: + negative_prompt = [""] * len(prompt) + + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + + input_ids = self.tokenizer( + negative_prompt, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=77, #self.tokenizer.model_max_length, + ).input_ids.to(self._execution_device) + + outputs = self.text_encoder(input_ids, return_dict=True, output_hidden_states=True) + negative_prompt_embeds = outputs.text_embeds + negative_encoder_hidden_states = outputs.hidden_states[-2] + + negative_prompt_embeds = negative_prompt_embeds.repeat(num_images_per_prompt, 1) + negative_encoder_hidden_states = negative_encoder_hidden_states.repeat(num_images_per_prompt, 1, 1) + + prompt_embeds = torch.concat([negative_prompt_embeds, prompt_embeds]) + encoder_hidden_states = torch.concat([negative_encoder_hidden_states, encoder_hidden_states]) + + image = self.image_processor.preprocess(image) + + height, width = image.shape[-2:] + + # Note that the micro conditionings _do_ flip the order of width, height for the original size + # and the crop coordinates. This is how it was done in the original code base + micro_conds = torch.tensor( + [ + width, + height, + micro_conditioning_crop_coord[0], + micro_conditioning_crop_coord[1], + micro_conditioning_aesthetic_score, + ], + device=self._execution_device, + dtype=encoder_hidden_states.dtype, + ) + + micro_conds = micro_conds.unsqueeze(0) + micro_conds = micro_conds.expand(2 * batch_size if guidance_scale > 1.0 else batch_size, -1) + + self.scheduler.set_timesteps(num_inference_steps, temperature, self._execution_device) + num_inference_steps = int(len(self.scheduler.timesteps) * strength) + start_timestep_idx = len(self.scheduler.timesteps) - num_inference_steps + + needs_upcasting = False # = self.vqvae.dtype == torch.float16 and self.vqvae.config.force_upcast + + if needs_upcasting: + self.vqvae.float() + + latents = self.vqvae.encode(image.to(dtype=self.vqvae.dtype, device=self._execution_device)).latents + latents_bsz, channels, latents_height, latents_width = latents.shape + latents = self.vqvae.quantize(latents)[2][2].reshape(latents_bsz, latents_height, latents_width) + latents = self.scheduler.add_noise( + latents, self.scheduler.timesteps[start_timestep_idx - 1], generator=generator + ) + latents = latents.repeat(num_images_per_prompt, 1, 1) + + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i in range(start_timestep_idx, len(self.scheduler.timesteps)): + timestep = self.scheduler.timesteps[i] + + if guidance_scale > 1.0: + model_input = torch.cat([latents] * 2) + else: + model_input = latents + if height == 1024: #args.resolution == 1024: + img_ids = _prepare_latent_image_ids(model_input.shape[0], model_input.shape[-2],model_input.shape[-1],model_input.device,model_input.dtype) + else: + img_ids = _prepare_latent_image_ids(model_input.shape[0],2*model_input.shape[-2],2*model_input.shape[-1],model_input.device,model_input.dtype) + txt_ids = torch.zeros(encoder_hidden_states.shape[1],3).to(device = encoder_hidden_states.device, dtype = encoder_hidden_states.dtype) + model_output = self.transformer( + model_input, + micro_conds=micro_conds, + pooled_projections=prompt_embeds, + encoder_hidden_states=encoder_hidden_states, + # cross_attention_kwargs=cross_attention_kwargs, + img_ids = img_ids, + txt_ids = txt_ids, + timestep = torch.tensor([timestep], device=model_input.device, dtype=torch.long), + ) + + if guidance_scale > 1.0: + uncond_logits, cond_logits = model_output.chunk(2) + model_output = uncond_logits + guidance_scale * (cond_logits - uncond_logits) + + latents = self.scheduler.step( + model_output=model_output, + timestep=timestep, + sample=latents, + generator=generator, + ).prev_sample + + if i == len(self.scheduler.timesteps) - 1 or ((i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, timestep, latents) + + if output_type == "latent": + output = latents + else: + output = self.vqvae.decode( + latents, + force_not_quantize=True, + shape=( + batch_size, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + self.vqvae.config.latent_channels, + ), + ).sample.clip(0, 1) + output = self.image_processor.postprocess(output, output_type) + + if needs_upcasting: + self.vqvae.half() + + self.maybe_free_model_hooks() + + if not return_dict: + return (output,) + + return ImagePipelineOutput(output) diff --git a/modules/meissonic/pipeline_inpaint.py b/modules/meissonic/pipeline_inpaint.py new file mode 100644 index 000000000..994846fba --- /dev/null +++ b/modules/meissonic/pipeline_inpaint.py @@ -0,0 +1,374 @@ +# Copyright 2024 The HuggingFace Team and The MeissonFlow Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +import torch +from transformers import CLIPTextModelWithProjection, CLIPTokenizer +from diffusers.image_processor import PipelineImageInput, VaeImageProcessor +from diffusers.models import VQModel +from diffusers.utils import replace_example_docstring +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from .scheduler import Scheduler +from .transformer import Transformer2DModel + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> pipe(prompt, input_image, mask).images[0].save("out.png") + ``` +""" + +def _prepare_latent_image_ids(batch_size, height, width, device, dtype): + latent_image_ids = torch.zeros(height // 2, width // 2, 3) + latent_image_ids[..., 1] = latent_image_ids[..., 1] + torch.arange(height // 2)[:, None] + latent_image_ids[..., 2] = latent_image_ids[..., 2] + torch.arange(width // 2)[None, :] + + latent_image_id_height, latent_image_id_width, latent_image_id_channels = latent_image_ids.shape + + latent_image_ids = latent_image_ids.reshape( + latent_image_id_height * latent_image_id_width, latent_image_id_channels + ) + # latent_image_ids = latent_image_ids.unsqueeze(0).repeat(batch_size, 1, 1) + + return latent_image_ids.to(device=device, dtype=dtype) + + +class InpaintPipeline(DiffusionPipeline): + image_processor: VaeImageProcessor + vqvae: VQModel + tokenizer: CLIPTokenizer + text_encoder: CLIPTextModelWithProjection + transformer: Transformer2DModel #UVit2DModel + scheduler: Scheduler + + model_cpu_offload_seq = "text_encoder->transformer->vqvae" + + # TODO - when calling self.vqvae.quantize, it uses self.vqvae.quantize.embedding.weight before + # the forward method of self.vqvae.quantize, so the hook doesn't get called to move the parameter + # off the meta device. There should be a way to fix this instead of just not offloading it + _exclude_from_cpu_offload = ["vqvae"] + + def __init__( + self, + vqvae: VQModel, + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModelWithProjection, + transformer: Transformer2DModel, #UVit2DModel, + scheduler: Scheduler, + ): + super().__init__() + + self.register_modules( + vqvae=vqvae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vqvae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_normalize=False) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, + do_normalize=False, + do_binarize=True, + do_convert_grayscale=True, + do_resize=True, + ) + self.scheduler.register_to_config(masking_schedule="linear") + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[List[str], str]] = None, + image: PipelineImageInput = None, + mask_image: PipelineImageInput = None, + strength: float = 1.0, + num_inference_steps: int = 12, + guidance_scale: float = 10.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[torch.Generator] = None, + prompt_embeds: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + negative_encoder_hidden_states: Optional[torch.Tensor] = None, + output_type="pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + micro_conditioning_aesthetic_score: int = 6, + micro_conditioning_crop_coord: Tuple[int, int] = (0, 0), + temperature: Union[int, Tuple[int, int], List[int]] = (2, 0), + ): + """ + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both + numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list + or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a + list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask + are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a + single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one + color channel (L) instead of 3, so the expected shape for pytorch tensor would be `(B, 1, H, W)`, `(B, + H, W)`, `(1, H, W)`, `(H, W)`. And for numpy array would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W, + 1)`, or `(H, W)`. + strength (`float`, *optional*, defaults to 1.0): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 16): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 10.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. A single vector from the + pooled and projected final hidden states. + encoder_hidden_states (`torch.Tensor`, *optional*): + Pre-generated penultimate hidden states from the text encoder providing additional text conditioning. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + negative_encoder_hidden_states (`torch.Tensor`, *optional*): + Analogous to `encoder_hidden_states` for the positive prompt. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + micro_conditioning_aesthetic_score (`int`, *optional*, defaults to 6): + The targeted aesthetic score according to the laion aesthetic classifier. See + https://laion.ai/blog/laion-aesthetics/ and the micro-conditioning section of + https://arxiv.org/abs/2307.01952. + micro_conditioning_crop_coord (`Tuple[int]`, *optional*, defaults to (0, 0)): + The targeted height, width crop coordinates. See the micro-conditioning section of + https://arxiv.org/abs/2307.01952. + temperature (`Union[int, Tuple[int, int], List[int]]`, *optional*, defaults to (2, 0)): + Configures the temperature scheduler on `self.scheduler` see `AmusedScheduler#set_timesteps`. + + Examples: + + Returns: + [`~pipelines.pipeline_utils.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.pipeline_utils.ImagePipelineOutput`] is returned, otherwise a + `tuple` is returned where the first element is a list with the generated images. + """ + + if (prompt_embeds is not None and encoder_hidden_states is None) or ( + prompt_embeds is None and encoder_hidden_states is not None + ): + raise ValueError("pass either both `prompt_embeds` and `encoder_hidden_states` or neither") + + if (negative_prompt_embeds is not None and negative_encoder_hidden_states is None) or ( + negative_prompt_embeds is None and negative_encoder_hidden_states is not None + ): + raise ValueError( + "pass either both `negatve_prompt_embeds` and `negative_encoder_hidden_states` or neither" + ) + + if (prompt is None and prompt_embeds is None) or (prompt is not None and prompt_embeds is not None): + raise ValueError("pass only one of `prompt` or `prompt_embeds`") + + if isinstance(prompt, str): + prompt = [prompt] + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + batch_size = batch_size * num_images_per_prompt + + if prompt_embeds is None: + input_ids = self.tokenizer( + prompt, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=77, #self.tokenizer.model_max_length, + ).input_ids.to(self._execution_device) + + outputs = self.text_encoder(input_ids, return_dict=True, output_hidden_states=True) + prompt_embeds = outputs.text_embeds + encoder_hidden_states = outputs.hidden_states[-2] + + prompt_embeds = prompt_embeds.repeat(num_images_per_prompt, 1) + encoder_hidden_states = encoder_hidden_states.repeat(num_images_per_prompt, 1, 1) + + if guidance_scale > 1.0: + if negative_prompt_embeds is None: + if negative_prompt is None: + negative_prompt = [""] * len(prompt) + + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + + input_ids = self.tokenizer( + negative_prompt, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=77, #self.tokenizer.model_max_length, + ).input_ids.to(self._execution_device) + + outputs = self.text_encoder(input_ids, return_dict=True, output_hidden_states=True) + negative_prompt_embeds = outputs.text_embeds + negative_encoder_hidden_states = outputs.hidden_states[-2] + + negative_prompt_embeds = negative_prompt_embeds.repeat(num_images_per_prompt, 1) + negative_encoder_hidden_states = negative_encoder_hidden_states.repeat(num_images_per_prompt, 1, 1) + + prompt_embeds = torch.concat([negative_prompt_embeds, prompt_embeds]) + encoder_hidden_states = torch.concat([negative_encoder_hidden_states, encoder_hidden_states]) + + image = self.image_processor.preprocess(image) + + height, width = image.shape[-2:] + + # Note that the micro conditionings _do_ flip the order of width, height for the original size + # and the crop coordinates. This is how it was done in the original code base + micro_conds = torch.tensor( + [ + width, + height, + micro_conditioning_crop_coord[0], + micro_conditioning_crop_coord[1], + micro_conditioning_aesthetic_score, + ], + device=self._execution_device, + dtype=encoder_hidden_states.dtype, + ) + + micro_conds = micro_conds.unsqueeze(0) + micro_conds = micro_conds.expand(2 * batch_size if guidance_scale > 1.0 else batch_size, -1) + + self.scheduler.set_timesteps(num_inference_steps, temperature, self._execution_device) + num_inference_steps = int(len(self.scheduler.timesteps) * strength) + start_timestep_idx = len(self.scheduler.timesteps) - num_inference_steps + + needs_upcasting = False #self.vqvae.dtype == torch.float16 and self.vqvae.config.force_upcast + + if needs_upcasting: + self.vqvae.float() + + latents = self.vqvae.encode(image.to(dtype=self.vqvae.dtype, device=self._execution_device)).latents + latents_bsz, channels, latents_height, latents_width = latents.shape + latents = self.vqvae.quantize(latents)[2][2].reshape(latents_bsz, latents_height, latents_width) + + mask = self.mask_processor.preprocess( + mask_image, height // self.vae_scale_factor, width // self.vae_scale_factor + ) + mask = mask.reshape(mask.shape[0], latents_height, latents_width).bool().to(latents.device) + latents[mask] = self.scheduler.config.mask_token_id + + starting_mask_ratio = mask.sum() / latents.numel() + + latents = latents.repeat(num_images_per_prompt, 1, 1) + + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i in range(start_timestep_idx, len(self.scheduler.timesteps)): + timestep = self.scheduler.timesteps[i] + + if guidance_scale > 1.0: + model_input = torch.cat([latents] * 2) + else: + model_input = latents + + if height == 1024: #args.resolution == 1024: + img_ids = _prepare_latent_image_ids(model_input.shape[0], model_input.shape[-2],model_input.shape[-1],model_input.device,model_input.dtype) + else: + img_ids = _prepare_latent_image_ids(model_input.shape[0],2*model_input.shape[-2],2*model_input.shape[-1],model_input.device,model_input.dtype) + txt_ids = torch.zeros(encoder_hidden_states.shape[1],3).to(device = encoder_hidden_states.device, dtype = encoder_hidden_states.dtype) + model_output = self.transformer( + model_input, + micro_conds=micro_conds, + pooled_projections=prompt_embeds, + encoder_hidden_states=encoder_hidden_states, + # cross_attention_kwargs=cross_attention_kwargs, + img_ids = img_ids, + txt_ids = txt_ids, + timestep = torch.tensor([timestep], device=model_input.device, dtype=torch.long), + ) + + if guidance_scale > 1.0: + uncond_logits, cond_logits = model_output.chunk(2) + model_output = uncond_logits + guidance_scale * (cond_logits - uncond_logits) + + latents = self.scheduler.step( + model_output=model_output, + timestep=timestep, + sample=latents, + generator=generator, + starting_mask_ratio=starting_mask_ratio, + ).prev_sample + + if i == len(self.scheduler.timesteps) - 1 or ((i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, timestep, latents) + + if output_type == "latent": + output = latents + else: + output = self.vqvae.decode( + latents, + force_not_quantize=True, + shape=( + batch_size, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + self.vqvae.config.latent_channels, + ), + ).sample.clip(0, 1) + output = self.image_processor.postprocess(output, output_type) + + if needs_upcasting: + self.vqvae.half() + + self.maybe_free_model_hooks() + + if not return_dict: + return (output,) + + return ImagePipelineOutput(output) diff --git a/modules/meissonic/scheduler.py b/modules/meissonic/scheduler.py new file mode 100644 index 000000000..3d2fe4276 --- /dev/null +++ b/modules/meissonic/scheduler.py @@ -0,0 +1,175 @@ +# Copyright 2024 The HuggingFace Team and The MeissonFlow Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.utils import BaseOutput +from diffusers.schedulers.scheduling_utils import SchedulerMixin + + +def gumbel_noise(t, generator=None): + device = generator.device if generator is not None else t.device + noise = torch.zeros_like(t, device=device).uniform_(0, 1, generator=generator).to(t.device) + return -torch.log((-torch.log(noise.clamp(1e-20))).clamp(1e-20)) + + +def mask_by_random_topk(mask_len, probs, temperature=1.0, generator=None): + confidence = torch.log(probs.clamp(1e-20)) + temperature * gumbel_noise(probs, generator=generator) + sorted_confidence = torch.sort(confidence, dim=-1).values + cut_off = torch.gather(sorted_confidence, 1, mask_len.long()) + masking = confidence < cut_off + return masking + + +@dataclass +class SchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.Tensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.Tensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.Tensor + pred_original_sample: torch.Tensor = None + + +class Scheduler(SchedulerMixin, ConfigMixin): + order = 1 + + temperatures: torch.Tensor + + @register_to_config + def __init__( + self, + mask_token_id: int, + masking_schedule: str = "cosine", + ): + self.temperatures = None + self.timesteps = None + + def set_timesteps( + self, + num_inference_steps: int, + temperature: Union[int, Tuple[int, int], List[int]] = (2, 0), + device: Union[str, torch.device] = None, + ): + self.timesteps = torch.arange(num_inference_steps, device=device).flip(0) + + if isinstance(temperature, (tuple, list)): + self.temperatures = torch.linspace(temperature[0], temperature[1], num_inference_steps, device=device) + else: + self.temperatures = torch.linspace(temperature, 0.01, num_inference_steps, device=device) + + def step( + self, + model_output: torch.Tensor, + timestep: torch.long, + sample: torch.LongTensor, + starting_mask_ratio: int = 1, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + two_dim_input = sample.ndim == 3 and model_output.ndim == 4 + + if two_dim_input: + batch_size, codebook_size, height, width = model_output.shape + sample = sample.reshape(batch_size, height * width) + model_output = model_output.reshape(batch_size, codebook_size, height * width).permute(0, 2, 1) + + unknown_map = sample == self.config.mask_token_id + + probs = model_output.softmax(dim=-1) + + device = probs.device + probs_ = probs.to(generator.device) if generator is not None else probs # handles when generator is on CPU + if probs_.device.type == "cpu" and probs_.dtype != torch.float32: + probs_ = probs_.float() # multinomial is not implemented for cpu half precision + probs_ = probs_.reshape(-1, probs.size(-1)) + pred_original_sample = torch.multinomial(probs_, 1, generator=generator).to(device=device) + pred_original_sample = pred_original_sample[:, 0].view(*probs.shape[:-1]) + pred_original_sample = torch.where(unknown_map, pred_original_sample, sample) + + if timestep == 0: + prev_sample = pred_original_sample + else: + seq_len = sample.shape[1] + step_idx = (self.timesteps == timestep).nonzero() + ratio = (step_idx + 1) / len(self.timesteps) + + if self.config.masking_schedule == "cosine": + mask_ratio = torch.cos(ratio * math.pi / 2) + elif self.config.masking_schedule == "linear": + mask_ratio = 1 - ratio + else: + raise ValueError(f"unknown masking schedule {self.config.masking_schedule}") + + mask_ratio = starting_mask_ratio * mask_ratio + + mask_len = (seq_len * mask_ratio).floor() + # do not mask more than amount previously masked + mask_len = torch.min(unknown_map.sum(dim=-1, keepdim=True) - 1, mask_len) + # mask at least one + mask_len = torch.max(torch.tensor([1], device=model_output.device), mask_len) + + selected_probs = torch.gather(probs, -1, pred_original_sample[:, :, None])[:, :, 0] + # Ignores the tokens given in the input by overwriting their confidence. + selected_probs = torch.where(unknown_map, selected_probs, torch.finfo(selected_probs.dtype).max) + + masking = mask_by_random_topk(mask_len, selected_probs, self.temperatures[step_idx], generator) + + # Masks tokens with lower confidence. + prev_sample = torch.where(masking, self.config.mask_token_id, pred_original_sample) + + if two_dim_input: + prev_sample = prev_sample.reshape(batch_size, height, width) + pred_original_sample = pred_original_sample.reshape(batch_size, height, width) + + if not return_dict: + return (prev_sample, pred_original_sample) + + return SchedulerOutput(prev_sample, pred_original_sample) + + def add_noise(self, sample, timesteps, generator=None): + step_idx = (self.timesteps == timesteps).nonzero() + ratio = (step_idx + 1) / len(self.timesteps) + + if self.config.masking_schedule == "cosine": + mask_ratio = torch.cos(ratio * math.pi / 2) + elif self.config.masking_schedule == "linear": + mask_ratio = 1 - ratio + else: + raise ValueError(f"unknown masking schedule {self.config.masking_schedule}") + + mask_indices = ( + torch.rand( + sample.shape, device=generator.device if generator is not None else sample.device, generator=generator + ).to(sample.device) + < mask_ratio + ) + + masked_sample = sample.clone() + + masked_sample[mask_indices] = self.config.mask_token_id + + return masked_sample diff --git a/modules/meissonic/test.py b/modules/meissonic/test.py new file mode 100644 index 000000000..46189c85d --- /dev/null +++ b/modules/meissonic/test.py @@ -0,0 +1,33 @@ +import sys +sys.path.append("./") + +# import torch +# from torchvision import transforms +from meissonic.transformer import Transformer2DModel as TransformerMeissonic +from meissonic.pipeline import Pipeline as PipelineMeissonic +from meissonic.scheduler import Scheduler as MeissonicScheduler +from transformers import CLIPTextModelWithProjection, CLIPTokenizer +from diffusers import VQModel + +device = 'cuda' +model_path = 'MeissonFlow/Meissonic' +cache_dir = '/mnt/models/Diffusers' + +# diffusers_load_config['variant'] = fp16 + +model = TransformerMeissonic.from_pretrained(model_path, subfolder="transformer", cache_dir=cache_dir) +vq_model = VQModel.from_pretrained(model_path, subfolder="vqvae", cache_dir=cache_dir) +# text_encoder = CLIPTextModelWithProjection.from_pretrained(model_path,subfolder="text_encoder",) +text_encoder = CLIPTextModelWithProjection.from_pretrained("laion/CLIP-ViT-H-14-laion2B-s32B-b79K", cache_dir=cache_dir) +tokenizer = CLIPTokenizer.from_pretrained(model_path, subfolder="tokenizer") +scheduler = MeissonicScheduler.from_pretrained(model_path, subfolder="scheduler") +pipe = PipelineMeissonic(vq_model, tokenizer=tokenizer, text_encoder=text_encoder, transformer=model, scheduler=scheduler) +pipe = pipe.to(device) + +steps = 64 +guidance_scale = 9 +resolution = 1024 +negative = "worst quality, low quality, low res, blurry, distortion, watermark, logo, signature, text, jpeg artifacts, signature, sketch, duplicate, ugly, identifying mark" +prompt = "Beautiful young woman posing on a lake with snow covered mountains in the background" +image = pipe(prompt=prompt, negative_prompt=negative, height=resolution, width=resolution, guidance_scale=guidance_scale, num_inference_steps=steps).images[0] +image.save('/tmp/meissonic.png') diff --git a/modules/meissonic/transformer.py b/modules/meissonic/transformer.py new file mode 100644 index 000000000..64f91baa2 --- /dev/null +++ b/modules/meissonic/transformer.py @@ -0,0 +1,1214 @@ +# Copyright 2024 Black Forest Labs, The HuggingFace Team, The InstantX Team and The MeissonFlow Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Any, Dict, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import FromOriginalModelMixin, PeftAdapterMixin +from diffusers.models.attention import FeedForward, BasicTransformerBlock, SkipFFTransformerBlock +from diffusers.models.attention_processor import ( + Attention, + AttentionProcessor, + FluxAttnProcessor2_0, + # FusedFluxAttnProcessor2_0, +) +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import AdaLayerNormContinuous, AdaLayerNormZero, AdaLayerNormZeroSingle, GlobalResponseNorm, RMSNorm +from diffusers.utils import USE_PEFT_BACKEND, is_torch_version, logging, scale_lora_layers, unscale_lora_layers +from diffusers.utils.torch_utils import maybe_allow_in_graph +from diffusers.models.embeddings import CombinedTimestepGuidanceTextProjEmbeddings, CombinedTimestepTextProjEmbeddings,TimestepEmbedding, get_timestep_embedding #,FluxPosEmbed +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.resnet import Downsample2D, Upsample2D + +from typing import List + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + + +def get_3d_rotary_pos_embed( + embed_dim, crops_coords, grid_size, temporal_size, theta: int = 10000, use_real: bool = True +) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """ + RoPE for video tokens with 3D structure. + + Args: + embed_dim: (`int`): + The embedding dimension size, corresponding to hidden_size_head. + crops_coords (`Tuple[int]`): + The top-left and bottom-right coordinates of the crop. + grid_size (`Tuple[int]`): + The grid size of the spatial positional embedding (height, width). + temporal_size (`int`): + The size of the temporal dimension. + theta (`float`): + Scaling factor for frequency computation. + use_real (`bool`): + If True, return real part and imaginary part separately. Otherwise, return complex numbers. + + Returns: + `torch.Tensor`: positional embedding with shape `(temporal_size * grid_size[0] * grid_size[1], embed_dim/2)`. + """ + start, stop = crops_coords + grid_h = np.linspace(start[0], stop[0], grid_size[0], endpoint=False, dtype=np.float32) + grid_w = np.linspace(start[1], stop[1], grid_size[1], endpoint=False, dtype=np.float32) + grid_t = np.linspace(0, temporal_size, temporal_size, endpoint=False, dtype=np.float32) + + # Compute dimensions for each axis + dim_t = embed_dim // 4 + dim_h = embed_dim // 8 * 3 + dim_w = embed_dim // 8 * 3 + + # Temporal frequencies + freqs_t = 1.0 / (theta ** (torch.arange(0, dim_t, 2).float() / dim_t)) + grid_t = torch.from_numpy(grid_t).float() + freqs_t = torch.einsum("n , f -> n f", grid_t, freqs_t) + freqs_t = freqs_t.repeat_interleave(2, dim=-1) + + # Spatial frequencies for height and width + freqs_h = 1.0 / (theta ** (torch.arange(0, dim_h, 2).float() / dim_h)) + freqs_w = 1.0 / (theta ** (torch.arange(0, dim_w, 2).float() / dim_w)) + grid_h = torch.from_numpy(grid_h).float() + grid_w = torch.from_numpy(grid_w).float() + freqs_h = torch.einsum("n , f -> n f", grid_h, freqs_h) + freqs_w = torch.einsum("n , f -> n f", grid_w, freqs_w) + freqs_h = freqs_h.repeat_interleave(2, dim=-1) + freqs_w = freqs_w.repeat_interleave(2, dim=-1) + + # Broadcast and concatenate tensors along specified dimension + def broadcast(tensors, dim=-1): + num_tensors = len(tensors) + shape_lens = {len(t.shape) for t in tensors} + assert len(shape_lens) == 1, "tensors must all have the same number of dimensions" + shape_len = list(shape_lens)[0] + dim = (dim + shape_len) if dim < 0 else dim + dims = list(zip(*(list(t.shape) for t in tensors))) + expandable_dims = [(i, val) for i, val in enumerate(dims) if i != dim] + assert all( + [*(len(set(t[1])) <= 2 for t in expandable_dims)] + ), "invalid dimensions for broadcastable concatenation" + max_dims = [(t[0], max(t[1])) for t in expandable_dims] + expanded_dims = [(t[0], (t[1],) * num_tensors) for t in max_dims] + expanded_dims.insert(dim, (dim, dims[dim])) + expandable_shapes = list(zip(*(t[1] for t in expanded_dims))) + tensors = [t[0].expand(*t[1]) for t in zip(tensors, expandable_shapes)] + return torch.cat(tensors, dim=dim) + + freqs = broadcast((freqs_t[:, None, None, :], freqs_h[None, :, None, :], freqs_w[None, None, :, :]), dim=-1) + + t, h, w, d = freqs.shape + freqs = freqs.view(t * h * w, d) + + # Generate sine and cosine components + sin = freqs.sin() + cos = freqs.cos() + + if use_real: + return cos, sin + else: + freqs_cis = torch.polar(torch.ones_like(freqs), freqs) + return freqs_cis + + +def get_2d_rotary_pos_embed(embed_dim, crops_coords, grid_size, use_real=True): + """ + RoPE for image tokens with 2d structure. + + Args: + embed_dim: (`int`): + The embedding dimension size + crops_coords (`Tuple[int]`) + The top-left and bottom-right coordinates of the crop. + grid_size (`Tuple[int]`): + The grid size of the positional embedding. + use_real (`bool`): + If True, return real part and imaginary part separately. Otherwise, return complex numbers. + + Returns: + `torch.Tensor`: positional embedding with shape `( grid_size * grid_size, embed_dim/2)`. + """ + start, stop = crops_coords + grid_h = np.linspace(start[0], stop[0], grid_size[0], endpoint=False, dtype=np.float32) + grid_w = np.linspace(start[1], stop[1], grid_size[1], endpoint=False, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) # here w goes first + grid = np.stack(grid, axis=0) # [2, W, H] + + grid = grid.reshape([2, 1, *grid.shape[1:]]) + pos_embed = get_2d_rotary_pos_embed_from_grid(embed_dim, grid, use_real=use_real) + return pos_embed + + +def get_2d_rotary_pos_embed_from_grid(embed_dim, grid, use_real=False): + assert embed_dim % 4 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_rotary_pos_embed( + embed_dim // 2, grid[0].reshape(-1), use_real=use_real + ) # (H*W, D/2) if use_real else (H*W, D/4) + emb_w = get_1d_rotary_pos_embed( + embed_dim // 2, grid[1].reshape(-1), use_real=use_real + ) # (H*W, D/2) if use_real else (H*W, D/4) + + if use_real: + cos = torch.cat([emb_h[0], emb_w[0]], dim=1) # (H*W, D) + sin = torch.cat([emb_h[1], emb_w[1]], dim=1) # (H*W, D) + return cos, sin + else: + emb = torch.cat([emb_h, emb_w], dim=1) # (H*W, D/2) + return emb + + +def get_2d_rotary_pos_embed_lumina(embed_dim, len_h, len_w, linear_factor=1.0, ntk_factor=1.0): + assert embed_dim % 4 == 0 + + emb_h = get_1d_rotary_pos_embed( + embed_dim // 2, len_h, linear_factor=linear_factor, ntk_factor=ntk_factor + ) # (H, D/4) + emb_w = get_1d_rotary_pos_embed( + embed_dim // 2, len_w, linear_factor=linear_factor, ntk_factor=ntk_factor + ) # (W, D/4) + emb_h = emb_h.view(len_h, 1, embed_dim // 4, 1).repeat(1, len_w, 1, 1) # (H, W, D/4, 1) + emb_w = emb_w.view(1, len_w, embed_dim // 4, 1).repeat(len_h, 1, 1, 1) # (H, W, D/4, 1) + + emb = torch.cat([emb_h, emb_w], dim=-1).flatten(2) # (H, W, D/2) + return emb + + +def get_1d_rotary_pos_embed( + dim: int, + pos: Union[np.ndarray, int], + theta: float = 10000.0, + use_real=False, + linear_factor=1.0, + ntk_factor=1.0, + repeat_interleave_real=True, + freqs_dtype=torch.float32, # torch.float32 (hunyuan, stable audio), torch.float64 (flux) +): + """ + Precompute the frequency tensor for complex exponentials (cis) with given dimensions. + + This function calculates a frequency tensor with complex exponentials using the given dimension 'dim' and the end + index 'end'. The 'theta' parameter scales the frequencies. The returned tensor contains complex values in complex64 + data type. + + Args: + dim (`int`): Dimension of the frequency tensor. + pos (`np.ndarray` or `int`): Position indices for the frequency tensor. [S] or scalar + theta (`float`, *optional*, defaults to 10000.0): + Scaling factor for frequency computation. Defaults to 10000.0. + use_real (`bool`, *optional*): + If True, return real part and imaginary part separately. Otherwise, return complex numbers. + linear_factor (`float`, *optional*, defaults to 1.0): + Scaling factor for the context extrapolation. Defaults to 1.0. + ntk_factor (`float`, *optional*, defaults to 1.0): + Scaling factor for the NTK-Aware RoPE. Defaults to 1.0. + repeat_interleave_real (`bool`, *optional*, defaults to `True`): + If `True` and `use_real`, real part and imaginary part are each interleaved with themselves to reach `dim`. + Otherwise, they are concateanted with themselves. + freqs_dtype (`torch.float32` or `torch.float64`, *optional*, defaults to `torch.float32`): + the dtype of the frequency tensor. + Returns: + `torch.Tensor`: Precomputed frequency tensor with complex exponentials. [S, D/2] + """ + assert dim % 2 == 0 + + if isinstance(pos, int): + pos = np.arange(pos) + theta = theta * ntk_factor + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=freqs_dtype)[: (dim // 2)] / dim)) / linear_factor # [D/2] + t = torch.from_numpy(pos).to(freqs.device) # type: ignore # [S] + freqs = torch.outer(t, freqs) # type: ignore # [S, D/2] + if use_real and repeat_interleave_real: + freqs_cos = freqs.cos().repeat_interleave(2, dim=1).float() # [S, D] + freqs_sin = freqs.sin().repeat_interleave(2, dim=1).float() # [S, D] + return freqs_cos, freqs_sin + elif use_real: + freqs_cos = torch.cat([freqs.cos(), freqs.cos()], dim=-1).float() # [S, D] + freqs_sin = torch.cat([freqs.sin(), freqs.sin()], dim=-1).float() # [S, D] + return freqs_cos, freqs_sin + else: + freqs_cis = torch.polar(torch.ones_like(freqs), freqs).float() # complex64 # [S, D/2] + return freqs_cis + + +class FluxPosEmbed(nn.Module): + # modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11 + def __init__(self, theta: int, axes_dim: List[int]): + super().__init__() + self.theta = theta + self.axes_dim = axes_dim + + def forward(self, ids: torch.Tensor) -> torch.Tensor: + n_axes = ids.shape[-1] + cos_out = [] + sin_out = [] + pos = ids.squeeze().float().cpu().numpy() + is_mps = ids.device.type == "mps" + freqs_dtype = torch.float32 if is_mps else torch.float64 + for i in range(n_axes): + cos, sin = get_1d_rotary_pos_embed( + self.axes_dim[i], pos[:, i], repeat_interleave_real=True, use_real=True, freqs_dtype=freqs_dtype + ) + cos_out.append(cos) + sin_out.append(sin) + freqs_cos = torch.cat(cos_out, dim=-1).to(ids.device) + freqs_sin = torch.cat(sin_out, dim=-1).to(ids.device) + return freqs_cos, freqs_sin + + + +class FusedFluxAttnProcessor2_0: + """Attention processor used typically in processing the SD3-like self-attention projections.""" + + def __init__(self): + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError( + "FusedFluxAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0." + ) + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: torch.FloatTensor = None, + attention_mask: Optional[torch.FloatTensor] = None, + image_rotary_emb: Optional[torch.Tensor] = None, + ) -> torch.FloatTensor: + batch_size, _, _ = hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + + # `sample` projections. + qkv = attn.to_qkv(hidden_states) + split_size = qkv.shape[-1] // 3 + query, key, value = torch.split(qkv, split_size, dim=-1) + + inner_dim = key.shape[-1] + head_dim = inner_dim // attn.heads + + query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + if attn.norm_q is not None: + query = attn.norm_q(query) + if attn.norm_k is not None: + key = attn.norm_k(key) + + # the attention in FluxSingleTransformerBlock does not use `encoder_hidden_states` + # `context` projections. + if encoder_hidden_states is not None: + encoder_qkv = attn.to_added_qkv(encoder_hidden_states) + split_size = encoder_qkv.shape[-1] // 3 + ( + encoder_hidden_states_query_proj, + encoder_hidden_states_key_proj, + encoder_hidden_states_value_proj, + ) = torch.split(encoder_qkv, split_size, dim=-1) + + encoder_hidden_states_query_proj = encoder_hidden_states_query_proj.view( + batch_size, -1, attn.heads, head_dim + ).transpose(1, 2) + encoder_hidden_states_key_proj = encoder_hidden_states_key_proj.view( + batch_size, -1, attn.heads, head_dim + ).transpose(1, 2) + encoder_hidden_states_value_proj = encoder_hidden_states_value_proj.view( + batch_size, -1, attn.heads, head_dim + ).transpose(1, 2) + + if attn.norm_added_q is not None: + encoder_hidden_states_query_proj = attn.norm_added_q(encoder_hidden_states_query_proj) + if attn.norm_added_k is not None: + encoder_hidden_states_key_proj = attn.norm_added_k(encoder_hidden_states_key_proj) + + # attention + query = torch.cat([encoder_hidden_states_query_proj, query], dim=2) + key = torch.cat([encoder_hidden_states_key_proj, key], dim=2) + value = torch.cat([encoder_hidden_states_value_proj, value], dim=2) + + # if image_rotary_emb is not None: # TODO broken import + # from .embeddings import apply_rotary_emb + # query = apply_rotary_emb(query, image_rotary_emb) + # key = apply_rotary_emb(key, image_rotary_emb) + + hidden_states = F.scaled_dot_product_attention(query, key, value, dropout_p=0.0, is_causal=False) + hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) + hidden_states = hidden_states.to(query.dtype) + + if encoder_hidden_states is not None: + encoder_hidden_states, hidden_states = ( + hidden_states[:, : encoder_hidden_states.shape[1]], + hidden_states[:, encoder_hidden_states.shape[1] :], + ) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + encoder_hidden_states = attn.to_add_out(encoder_hidden_states) + + return hidden_states, encoder_hidden_states + else: + return hidden_states + + + +@maybe_allow_in_graph +class SingleTransformerBlock(nn.Module): + r""" + A Transformer block following the MMDiT architecture, introduced in Stable Diffusion 3. + + Reference: https://arxiv.org/abs/2403.03206 + + Parameters: + dim (`int`): The number of channels in the input and output. + num_attention_heads (`int`): The number of heads to use for multi-head attention. + attention_head_dim (`int`): The number of channels in each head. + context_pre_only (`bool`): Boolean to determine if we should add some blocks associated with the + processing of `context` conditions. + """ + + def __init__(self, dim, num_attention_heads, attention_head_dim, mlp_ratio=4.0): + super().__init__() + self.mlp_hidden_dim = int(dim * mlp_ratio) + + self.norm = AdaLayerNormZeroSingle(dim) + self.proj_mlp = nn.Linear(dim, self.mlp_hidden_dim) + self.act_mlp = nn.GELU(approximate="tanh") + self.proj_out = nn.Linear(dim + self.mlp_hidden_dim, dim) + + processor = FluxAttnProcessor2_0() + self.attn = Attention( + query_dim=dim, + cross_attention_dim=None, + dim_head=attention_head_dim, + heads=num_attention_heads, + out_dim=dim, + bias=True, + processor=processor, + qk_norm="rms_norm", + eps=1e-6, + pre_only=True, + ) + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: torch.FloatTensor, + image_rotary_emb=None, + ): + residual = hidden_states + norm_hidden_states, gate = self.norm(hidden_states, emb=temb) + mlp_hidden_states = self.act_mlp(self.proj_mlp(norm_hidden_states)) + + attn_output = self.attn( + hidden_states=norm_hidden_states, + image_rotary_emb=image_rotary_emb, + ) + + hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2) + gate = gate.unsqueeze(1) + hidden_states = gate * self.proj_out(hidden_states) + hidden_states = residual + hidden_states + if hidden_states.dtype == torch.float16: + hidden_states = hidden_states.clip(-65504, 65504) + + return hidden_states + +@maybe_allow_in_graph +class TransformerBlock(nn.Module): + r""" + A Transformer block following the MMDiT architecture, introduced in Stable Diffusion 3. + + Reference: https://arxiv.org/abs/2403.03206 + + Parameters: + dim (`int`): The number of channels in the input and output. + num_attention_heads (`int`): The number of heads to use for multi-head attention. + attention_head_dim (`int`): The number of channels in each head. + context_pre_only (`bool`): Boolean to determine if we should add some blocks associated with the + processing of `context` conditions. + """ + + def __init__(self, dim, num_attention_heads, attention_head_dim, qk_norm="rms_norm", eps=1e-6): + super().__init__() + + self.norm1 = AdaLayerNormZero(dim) + + self.norm1_context = AdaLayerNormZero(dim) + + if hasattr(F, "scaled_dot_product_attention"): + processor = FluxAttnProcessor2_0() + else: + raise ValueError( + "The current PyTorch version does not support the `scaled_dot_product_attention` function." + ) + self.attn = Attention( + query_dim=dim, + cross_attention_dim=None, + added_kv_proj_dim=dim, + dim_head=attention_head_dim, + heads=num_attention_heads, + out_dim=dim, + context_pre_only=False, + bias=True, + processor=processor, + qk_norm=qk_norm, + eps=eps, + ) + + self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + self.ff = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate") + # self.ff = FeedForward(dim=dim, dim_out=dim, activation_fn="swiglu") + + self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + self.ff_context = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate") + # self.ff_context = FeedForward(dim=dim, dim_out=dim, activation_fn="swiglu") + + # let chunk size default to None + self._chunk_size = None + self._chunk_dim = 0 + + def forward( + self, + hidden_states: torch.FloatTensor, + encoder_hidden_states: torch.FloatTensor, + temb: torch.FloatTensor, + image_rotary_emb=None, + ): + norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states, emb=temb) + + norm_encoder_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.norm1_context( + encoder_hidden_states, emb=temb + ) + # Attention. + attn_output, context_attn_output = self.attn( + hidden_states=norm_hidden_states, + encoder_hidden_states=norm_encoder_hidden_states, + image_rotary_emb=image_rotary_emb, + ) + + # Process attention outputs for the `hidden_states`. + attn_output = gate_msa.unsqueeze(1) * attn_output + hidden_states = hidden_states + attn_output + + norm_hidden_states = self.norm2(hidden_states) + norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None] + + ff_output = self.ff(norm_hidden_states) + ff_output = gate_mlp.unsqueeze(1) * ff_output + + hidden_states = hidden_states + ff_output + + # Process attention outputs for the `encoder_hidden_states`. + + context_attn_output = c_gate_msa.unsqueeze(1) * context_attn_output + encoder_hidden_states = encoder_hidden_states + context_attn_output + + norm_encoder_hidden_states = self.norm2_context(encoder_hidden_states) + norm_encoder_hidden_states = norm_encoder_hidden_states * (1 + c_scale_mlp[:, None]) + c_shift_mlp[:, None] + + context_ff_output = self.ff_context(norm_encoder_hidden_states) + encoder_hidden_states = encoder_hidden_states + c_gate_mlp.unsqueeze(1) * context_ff_output + if encoder_hidden_states.dtype == torch.float16: + encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504) + + return encoder_hidden_states, hidden_states + + +class UVit2DConvEmbed(nn.Module): + def __init__(self, in_channels, block_out_channels, vocab_size, elementwise_affine, eps, bias): + super().__init__() + self.embeddings = nn.Embedding(vocab_size, in_channels) + self.layer_norm = RMSNorm(in_channels, eps, elementwise_affine) + self.conv = nn.Conv2d(in_channels, block_out_channels, kernel_size=1, bias=bias) + + def forward(self, input_ids): + embeddings = self.embeddings(input_ids) + embeddings = self.layer_norm(embeddings) + embeddings = embeddings.permute(0, 3, 1, 2) + embeddings = self.conv(embeddings) + return embeddings + +class ConvMlmLayer(nn.Module): + def __init__( + self, + block_out_channels: int, + in_channels: int, + use_bias: bool, + ln_elementwise_affine: bool, + layer_norm_eps: float, + codebook_size: int, + ): + super().__init__() + self.conv1 = nn.Conv2d(block_out_channels, in_channels, kernel_size=1, bias=use_bias) + self.layer_norm = RMSNorm(in_channels, layer_norm_eps, ln_elementwise_affine) + self.conv2 = nn.Conv2d(in_channels, codebook_size, kernel_size=1, bias=use_bias) + + def forward(self, hidden_states): + hidden_states = self.conv1(hidden_states) + hidden_states = self.layer_norm(hidden_states.permute(0, 2, 3, 1)).permute(0, 3, 1, 2) + logits = self.conv2(hidden_states) + return logits + +class SwiGLU(nn.Module): + r""" + A [variant](https://arxiv.org/abs/2002.05202) of the gated linear unit activation function. It's similar to `GEGLU` + but uses SiLU / Swish instead of GeLU. + + Parameters: + dim_in (`int`): The number of channels in the input. + dim_out (`int`): The number of channels in the output. + bias (`bool`, defaults to True): Whether to use a bias in the linear layer. + """ + + def __init__(self, dim_in: int, dim_out: int, bias: bool = True): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out * 2, bias=bias) + self.activation = nn.SiLU() + + def forward(self, hidden_states): + hidden_states = self.proj(hidden_states) + hidden_states, gate = hidden_states.chunk(2, dim=-1) + return hidden_states * self.activation(gate) + +class ConvNextBlock(nn.Module): + def __init__( + self, channels, layer_norm_eps, ln_elementwise_affine, use_bias, hidden_dropout, hidden_size, res_ffn_factor=4 + ): + super().__init__() + self.depthwise = nn.Conv2d( + channels, + channels, + kernel_size=3, + padding=1, + groups=channels, + bias=use_bias, + ) + self.norm = RMSNorm(channels, layer_norm_eps, ln_elementwise_affine) + self.channelwise_linear_1 = nn.Linear(channels, int(channels * res_ffn_factor), bias=use_bias) + self.channelwise_act = nn.GELU() + self.channelwise_norm = GlobalResponseNorm(int(channels * res_ffn_factor)) + self.channelwise_linear_2 = nn.Linear(int(channels * res_ffn_factor), channels, bias=use_bias) + self.channelwise_dropout = nn.Dropout(hidden_dropout) + self.cond_embeds_mapper = nn.Linear(hidden_size, channels * 2, use_bias) + + def forward(self, x, cond_embeds): + x_res = x + + x = self.depthwise(x) + + x = x.permute(0, 2, 3, 1) + x = self.norm(x) + + x = self.channelwise_linear_1(x) + x = self.channelwise_act(x) + x = self.channelwise_norm(x) + x = self.channelwise_linear_2(x) + x = self.channelwise_dropout(x) + + x = x.permute(0, 3, 1, 2) + + x = x + x_res + + scale, shift = self.cond_embeds_mapper(F.silu(cond_embeds)).chunk(2, dim=1) + x = x * (1 + scale[:, :, None, None]) + shift[:, :, None, None] + + return x + +class Simple_UVitBlock(nn.Module): + def __init__( + self, + channels, + ln_elementwise_affine, + layer_norm_eps, + use_bias, + downsample: bool, + upsample: bool, + ): + super().__init__() + + if downsample: + self.downsample = Downsample2D( + channels, + use_conv=True, + padding=0, + name="Conv2d_0", + kernel_size=2, + norm_type="rms_norm", + eps=layer_norm_eps, + elementwise_affine=ln_elementwise_affine, + bias=use_bias, + ) + else: + self.downsample = None + + if upsample: + self.upsample = Upsample2D( + channels, + use_conv_transpose=True, + kernel_size=2, + padding=0, + name="conv", + norm_type="rms_norm", + eps=layer_norm_eps, + elementwise_affine=ln_elementwise_affine, + bias=use_bias, + interpolate=False, + ) + else: + self.upsample = None + + def forward(self, x): + # print("before,", x.shape) + if self.downsample is not None: + # print('downsample') + x = self.downsample(x) + + if self.upsample is not None: + # print('upsample') + x = self.upsample(x) + # print("after,", x.shape) + return x + + +class UVitBlock(nn.Module): + def __init__( + self, + channels, + num_res_blocks: int, + hidden_size, + hidden_dropout, + ln_elementwise_affine, + layer_norm_eps, + use_bias, + block_num_heads, + attention_dropout, + downsample: bool, + upsample: bool, + ): + super().__init__() + + if downsample: + self.downsample = Downsample2D( + channels, + use_conv=True, + padding=0, + name="Conv2d_0", + kernel_size=2, + norm_type="rms_norm", + eps=layer_norm_eps, + elementwise_affine=ln_elementwise_affine, + bias=use_bias, + ) + else: + self.downsample = None + + self.res_blocks = nn.ModuleList( + [ + ConvNextBlock( + channels, + layer_norm_eps, + ln_elementwise_affine, + use_bias, + hidden_dropout, + hidden_size, + ) + for i in range(num_res_blocks) + ] + ) + + self.attention_blocks = nn.ModuleList( + [ + SkipFFTransformerBlock( + channels, + block_num_heads, + channels // block_num_heads, + hidden_size, + use_bias, + attention_dropout, + channels, + attention_bias=use_bias, + attention_out_bias=use_bias, + ) + for _ in range(num_res_blocks) + ] + ) + + if upsample: + self.upsample = Upsample2D( + channels, + use_conv_transpose=True, + kernel_size=2, + padding=0, + name="conv", + norm_type="rms_norm", + eps=layer_norm_eps, + elementwise_affine=ln_elementwise_affine, + bias=use_bias, + interpolate=False, + ) + else: + self.upsample = None + + def forward(self, x, pooled_text_emb, encoder_hidden_states, cross_attention_kwargs): + if self.downsample is not None: + x = self.downsample(x) + + for res_block, attention_block in zip(self.res_blocks, self.attention_blocks): + x = res_block(x, pooled_text_emb) + + batch_size, channels, height, width = x.shape + x = x.view(batch_size, channels, height * width).permute(0, 2, 1) + x = attention_block( + x, encoder_hidden_states=encoder_hidden_states, cross_attention_kwargs=cross_attention_kwargs + ) + x = x.permute(0, 2, 1).view(batch_size, channels, height, width) + + if self.upsample is not None: + x = self.upsample(x) + + return x + +class Transformer2DModel(ModelMixin, ConfigMixin, PeftAdapterMixin, FromOriginalModelMixin): + """ + The Transformer model introduced in Flux. + + Reference: https://blackforestlabs.ai/announcing-black-forest-labs/ + + Parameters: + patch_size (`int`): Patch size to turn the input data into small patches. + in_channels (`int`, *optional*, defaults to 16): The number of channels in the input. + num_layers (`int`, *optional*, defaults to 18): The number of layers of MMDiT blocks to use. + num_single_layers (`int`, *optional*, defaults to 18): The number of layers of single DiT blocks to use. + attention_head_dim (`int`, *optional*, defaults to 64): The number of channels in each head. + num_attention_heads (`int`, *optional*, defaults to 18): The number of heads to use for multi-head attention. + joint_attention_dim (`int`, *optional*): The number of `encoder_hidden_states` dimensions to use. + pooled_projection_dim (`int`): Number of dimensions to use when projecting the `pooled_projections`. + guidance_embeds (`bool`, defaults to False): Whether to use guidance embeddings. + """ + + _supports_gradient_checkpointing = False #True + # Due to NotImplementedError: DDPOptimizer backend: Found a higher order op in the graph. This is not supported. Please turn off DDP optimizer using torch._dynamo.config.optimize_ddp=False. Note that this can cause performance degradation because there will be one bucket for the entire Dynamo graph. + # Please refer to this issue - https://github.com/pytorch/pytorch/issues/104674. + _no_split_modules = ["TransformerBlock", "SingleTransformerBlock"] + + @register_to_config + def __init__( + self, + patch_size: int = 1, + in_channels: int = 64, + num_layers: int = 19, + num_single_layers: int = 38, + attention_head_dim: int = 128, + num_attention_heads: int = 24, + joint_attention_dim: int = 4096, + pooled_projection_dim: int = 768, + guidance_embeds: bool = False, # unused in our implementation + axes_dims_rope: Tuple[int] = (16, 56, 56), + vocab_size: int = 8256, + codebook_size: int = 8192, + downsample: bool = False, + upsample: bool = False, + ): + super().__init__() + self.out_channels = in_channels + self.inner_dim = self.config.num_attention_heads * self.config.attention_head_dim + + self.pos_embed = FluxPosEmbed(theta=10000, axes_dim=axes_dims_rope) + text_time_guidance_cls = ( + CombinedTimestepGuidanceTextProjEmbeddings if guidance_embeds else CombinedTimestepTextProjEmbeddings + ) + self.time_text_embed = text_time_guidance_cls( + embedding_dim=self.inner_dim, pooled_projection_dim=self.config.pooled_projection_dim + ) + + self.context_embedder = nn.Linear(self.config.joint_attention_dim, self.inner_dim) + + self.transformer_blocks = nn.ModuleList( + [ + TransformerBlock( + dim=self.inner_dim, + num_attention_heads=self.config.num_attention_heads, + attention_head_dim=self.config.attention_head_dim, + ) + for i in range(self.config.num_layers) + ] + ) + + self.single_transformer_blocks = nn.ModuleList( + [ + SingleTransformerBlock( + dim=self.inner_dim, + num_attention_heads=self.config.num_attention_heads, + attention_head_dim=self.config.attention_head_dim, + ) + for i in range(self.config.num_single_layers) + ] + ) + + + self.gradient_checkpointing = False + + in_channels_embed = self.inner_dim + ln_elementwise_affine = True + layer_norm_eps = 1e-06 + use_bias = False + micro_cond_embed_dim = 1280 + self.embed = UVit2DConvEmbed( + in_channels_embed, self.inner_dim, self.config.vocab_size, ln_elementwise_affine, layer_norm_eps, use_bias + ) + self.mlm_layer = ConvMlmLayer( + self.inner_dim, in_channels_embed, use_bias, ln_elementwise_affine, layer_norm_eps, self.config.codebook_size + ) + self.cond_embed = TimestepEmbedding( + micro_cond_embed_dim + self.config.pooled_projection_dim, self.inner_dim, sample_proj_bias=use_bias + ) + self.encoder_proj_layer_norm = RMSNorm(self.inner_dim, layer_norm_eps, ln_elementwise_affine) + self.project_to_hidden_norm = RMSNorm(in_channels_embed, layer_norm_eps, ln_elementwise_affine) + self.project_to_hidden = nn.Linear(in_channels_embed, self.inner_dim, bias=use_bias) + self.project_from_hidden_norm = RMSNorm(self.inner_dim, layer_norm_eps, ln_elementwise_affine) + self.project_from_hidden = nn.Linear(self.inner_dim, in_channels_embed, bias=use_bias) + + self.down_block = Simple_UVitBlock( + self.inner_dim, + ln_elementwise_affine, + layer_norm_eps, + use_bias, + downsample, + False, + ) + self.up_block = Simple_UVitBlock( + self.inner_dim, #block_out_channels, + ln_elementwise_affine, + layer_norm_eps, + use_bias, + False, + upsample=upsample, + ) + + # self.fuse_qkv_projections() + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor() + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.fuse_qkv_projections with FusedAttnProcessor2_0->FusedFluxAttnProcessor2_0 + def fuse_qkv_projections(self): + """ + Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, key, value) + are fused. For cross-attention modules, key and value projection matrices are fused. + + + + This API is 🧪 experimental. + + + """ + self.original_attn_processors = None + + for _, attn_processor in self.attn_processors.items(): + if "Added" in str(attn_processor.__class__.__name__): + raise ValueError("`fuse_qkv_projections()` is not supported for models having added KV projections.") + + self.original_attn_processors = self.attn_processors + + for module in self.modules(): + if isinstance(module, Attention): + module.fuse_projections(fuse=True) + + self.set_attn_processor(FusedFluxAttnProcessor2_0()) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.unfuse_qkv_projections + def unfuse_qkv_projections(self): + """Disables the fused QKV projection if enabled. + + + + This API is 🧪 experimental. + + + + """ + if self.original_attn_processors is not None: + self.set_attn_processor(self.original_attn_processors) + + def _set_gradient_checkpointing(self, module, value=False): + if hasattr(module, "gradient_checkpointing"): + module.gradient_checkpointing = value + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor = None, + pooled_projections: torch.Tensor = None, + timestep: torch.LongTensor = None, + img_ids: torch.Tensor = None, + txt_ids: torch.Tensor = None, + guidance: torch.Tensor = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + controlnet_block_samples= None, + controlnet_single_block_samples=None, + return_dict: bool = True, + micro_conds: torch.Tensor = None, + ) -> Union[torch.FloatTensor, Transformer2DModelOutput]: + """ + The [`FluxTransformer2DModel`] forward method. + + Args: + hidden_states (`torch.FloatTensor` of shape `(batch size, channel, height, width)`): + Input `hidden_states`. + encoder_hidden_states (`torch.FloatTensor` of shape `(batch size, sequence_len, embed_dims)`): + Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. + pooled_projections (`torch.FloatTensor` of shape `(batch_size, projection_dim)`): Embeddings projected + from the embeddings of input conditions. + timestep ( `torch.LongTensor`): + Used to indicate denoising step. + block_controlnet_hidden_states: (`list` of `torch.Tensor`): + A list of tensors that if specified are added to the residuals of transformer blocks. + joint_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain + tuple. + + Returns: + If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a + `tuple` where the first element is the sample tensor. + """ + micro_cond_encode_dim = 256 # same as self.config.micro_cond_encode_dim = 256 from amused + micro_cond_embeds = get_timestep_embedding( + micro_conds.flatten(), micro_cond_encode_dim, flip_sin_to_cos=True, downscale_freq_shift=0 + ) + micro_cond_embeds = micro_cond_embeds.reshape((hidden_states.shape[0], -1)) + + pooled_projections = torch.cat([pooled_projections, micro_cond_embeds], dim=1) + pooled_projections = pooled_projections.to(dtype=self.dtype) + pooled_projections = self.cond_embed(pooled_projections).to(encoder_hidden_states.dtype) + + + hidden_states = self.embed(hidden_states) + + encoder_hidden_states = self.context_embedder(encoder_hidden_states) + encoder_hidden_states = self.encoder_proj_layer_norm(encoder_hidden_states) + hidden_states = self.down_block(hidden_states) + + batch_size, channels, height, width = hidden_states.shape + hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch_size, height * width, channels) + hidden_states = self.project_to_hidden_norm(hidden_states) + hidden_states = self.project_to_hidden(hidden_states) + + + if joint_attention_kwargs is not None: + joint_attention_kwargs = joint_attention_kwargs.copy() + lora_scale = joint_attention_kwargs.pop("scale", 1.0) + else: + lora_scale = 1.0 + + if USE_PEFT_BACKEND: + # weight the lora layers by setting `lora_scale` for each PEFT layer + scale_lora_layers(self, lora_scale) + else: + if joint_attention_kwargs is not None and joint_attention_kwargs.get("scale", None) is not None: + logger.warning( + "Passing `scale` via `joint_attention_kwargs` when not using the PEFT backend is ineffective." + ) + + timestep = timestep.to(hidden_states.dtype) * 1000 + if guidance is not None: + guidance = guidance.to(hidden_states.dtype) * 1000 + else: + guidance = None + temb = ( + self.time_text_embed(timestep, pooled_projections) + if guidance is None + else self.time_text_embed(timestep, guidance, pooled_projections) + ) + + if txt_ids.ndim == 3: + logger.warning( + "Passing `txt_ids` 3d torch.Tensor is deprecated." + "Please remove the batch dimension and pass it as a 2d torch Tensor" + ) + txt_ids = txt_ids[0] + if img_ids.ndim == 3: + logger.warning( + "Passing `img_ids` 3d torch.Tensor is deprecated." + "Please remove the batch dimension and pass it as a 2d torch Tensor" + ) + img_ids = img_ids[0] + ids = torch.cat((txt_ids, img_ids), dim=0) + + image_rotary_emb = self.pos_embed(ids) + + for index_block, block in enumerate(self.transformer_blocks): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + encoder_hidden_states, hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + encoder_hidden_states, + temb, + image_rotary_emb, + **ckpt_kwargs, + ) + + else: + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + ) + + + # controlnet residual + if controlnet_block_samples is not None: + interval_control = len(self.transformer_blocks) / len(controlnet_block_samples) + interval_control = int(np.ceil(interval_control)) + hidden_states = hidden_states + controlnet_block_samples[index_block // interval_control] + + hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1) + + for index_block, block in enumerate(self.single_transformer_blocks): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + temb, + image_rotary_emb, + **ckpt_kwargs, + ) + + else: + hidden_states = block( + hidden_states=hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + ) + + # controlnet residual + if controlnet_single_block_samples is not None: + interval_control = len(self.single_transformer_blocks) / len(controlnet_single_block_samples) + interval_control = int(np.ceil(interval_control)) + hidden_states[:, encoder_hidden_states.shape[1] :, ...] = ( + hidden_states[:, encoder_hidden_states.shape[1] :, ...] + + controlnet_single_block_samples[index_block // interval_control] + ) + + hidden_states = hidden_states[:, encoder_hidden_states.shape[1] :, ...] + + + hidden_states = self.project_from_hidden_norm(hidden_states) + hidden_states = self.project_from_hidden(hidden_states) + + + hidden_states = hidden_states.reshape(batch_size, height, width, channels).permute(0, 3, 1, 2) + + hidden_states = self.up_block(hidden_states) + + if USE_PEFT_BACKEND: + # remove `lora_scale` from each PEFT layer + unscale_lora_layers(self, lora_scale) + + output = self.mlm_layer(hidden_states) + # self.unfuse_qkv_projections() + if not return_dict: + return (output,) + + + return output \ No newline at end of file diff --git a/modules/model_meissonic.py b/modules/model_meissonic.py new file mode 100644 index 000000000..69ceab458 --- /dev/null +++ b/modules/model_meissonic.py @@ -0,0 +1,37 @@ +import transformers +import diffusers + + +def load_meissonic(checkpoint_info, diffusers_load_config={}): + from modules import shared, devices, modelloader, sd_models + from modules.meissonic.transformer import Transformer2DModel as TransformerMeissonic + from modules.meissonic.scheduler import Scheduler as MeissonicScheduler + from modules.meissonic.pipeline import Pipeline as PipelineMeissonic + from modules.meissonic.pipeline_img2img import Img2ImgPipeline as PipelineMeissonicImg2Img + from modules.meissonic.pipeline_inpaint import InpaintPipeline as PipelineMeissonicInpaint + + modelloader.hf_login() + fn = sd_models.path_to_repo(checkpoint_info.path) + cache_dir = shared.opts.diffusers_dir + + diffusers_load_config['variant'] = 'fp16' + diffusers_load_config['trust_remote_code'] = True + model = TransformerMeissonic.from_pretrained(fn, subfolder="transformer", cache_dir=cache_dir, **diffusers_load_config) + vqvae = diffusers.VQModel.from_pretrained(fn, subfolder="vqvae", cache_dir=cache_dir, **diffusers_load_config) + text_encoder = transformers.CLIPTextModelWithProjection.from_pretrained(fn, subfolder="text_encoder", cache_dir=cache_dir) + # text_encoder = transformers.CLIPTextModelWithProjection.from_pretrained("laion/CLIP-ViT-H-14-laion2B-s32B-b79K", cache_dir=cache_dir) + tokenizer = transformers.CLIPTokenizer.from_pretrained(fn, subfolder="tokenizer", cache_dir=cache_dir) + scheduler = MeissonicScheduler.from_pretrained(fn, subfolder="scheduler", cache_dir=cache_dir) + pipe = PipelineMeissonic( + vqvae=vqvae.to(devices.dtype), + text_encoder=text_encoder.to(devices.dtype), + transformer=model.to(devices.dtype), + tokenizer=tokenizer, + scheduler=scheduler, + ) + + diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["meissonic"] = PipelineMeissonic + diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["meissonic"] = PipelineMeissonicImg2Img + diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["meissonic"] = PipelineMeissonicInpaint + devices.torch_gc() + return pipe diff --git a/modules/sd_models.py b/modules/sd_models.py index bf1f79765..fa04c6a8e 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -563,6 +563,7 @@ def detect_pipeline(f: str, op: str = 'model', warning=True, quiet=False): if guess == 'Autodetect': try: guess = 'Stable Diffusion XL' if 'XL' in f.upper() else 'Stable Diffusion' + pipeline = None # guess by size if os.path.isfile(f) and f.endswith('.safetensors'): size = round(os.path.getsize(f) / 1024 / 1024) @@ -624,6 +625,9 @@ def detect_pipeline(f: str, op: str = 'model', warning=True, quiet=False): guess = 'AuraFlow' if 'cogview' in f.lower(): guess = 'CogView' + if 'meissonic' in f.lower(): + guess = 'Meissonic' + pipeline = 'custom' if 'flux' in f.lower(): guess = 'FLUX' if size > 11000 and size < 20000: @@ -638,18 +642,20 @@ def detect_pipeline(f: str, op: str = 'model', warning=True, quiet=False): elif guess == 'Stable Diffusion XL' and 'instruct' in f.lower(): guess = 'Stable Diffusion XL Instruct' # get actual pipeline - pipeline = shared_items.get_pipelines().get(guess, None) + pipeline = shared_items.get_pipelines().get(guess, None) if pipeline is None else pipeline if not quiet: - shared.log.info(f'Autodetect {op}: detect="{guess}" class={pipeline.__name__} file="{f}" size={size}MB') + shared.log.info(f'Autodetect {op}: detect="{guess}" class={getattr(pipeline, "__name__", None)} file="{f}" size={size}MB') except Exception as e: shared.log.error(f'Autodetect {op}: file="{f}" {e}') + if debug_load: + errors.display(e, f'Load {op}: {f}') return None, None else: try: size = round(os.path.getsize(f) / 1024 / 1024) - pipeline = shared_items.get_pipelines().get(guess, None) + pipeline = shared_items.get_pipelines().get(guess, None) if pipeline is None else pipeline if not quiet: - shared.log.info(f'Load {op}: detect="{guess}" class={pipeline.__name__} file="{f}" size={size}MB') + shared.log.info(f'Load {op}: detect="{guess}" class={getattr(pipeline, "__name__", None)} file="{f}" size={size}MB') except Exception as e: shared.log.error(f'Load {op}: detect="{guess}" file="{f}" {e}') @@ -1099,149 +1105,108 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No files = shared.walk_files(checkpoint_info.path, ['.safetensors', '.bin', '.ckpt']) if 'variant' not in diffusers_load_config and any('diffusion_pytorch_model.fp16' in f for f in files): # deal with diffusers lack of variant fallback when loading diffusers_load_config['variant'] = 'fp16' - if model_type in ['Stable Cascade']: # forced pipeline + if sd_model is None: try: - from modules.model_stablecascade import load_cascade_combined - sd_model = load_cascade_combined(checkpoint_info, diffusers_load_config) + if model_type in ['Stable Cascade']: # forced pipeline + from modules.model_stablecascade import load_cascade_combined + sd_model = load_cascade_combined(checkpoint_info, diffusers_load_config) + elif model_type in ['InstaFlow']: # forced pipeline + pipeline = diffusers.utils.get_class_from_dynamic_module('instaflow_one_step', module_file='pipeline.py') + sd_model = pipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + elif model_type in ['SegMoE']: # forced pipeline + from modules.segmoe.segmoe_model import SegMoEPipeline + sd_model = SegMoEPipeline(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + sd_model = sd_model.pipe # segmoe pipe does its stuff in __init__ and __call__ is the original pipeline + elif model_type in ['PixArt-Sigma']: # forced pipeline + from modules.model_pixart import load_pixart + sd_model = load_pixart(checkpoint_info, diffusers_load_config) + elif model_type in ['Lumina-Next']: # forced pipeline + from modules.model_lumina import load_lumina + sd_model = load_lumina(checkpoint_info, diffusers_load_config) + elif model_type in ['Kolors']: # forced pipeline + from modules.model_kolors import load_kolors + sd_model = load_kolors(checkpoint_info, diffusers_load_config) + elif model_type in ['AuraFlow']: # forced pipeline + from modules.model_auraflow import load_auraflow + sd_model = load_auraflow(checkpoint_info, diffusers_load_config) + elif model_type in ['FLUX']: + from modules.model_flux import load_flux + sd_model = load_flux(checkpoint_info, diffusers_load_config) + elif model_type in ['Stable Diffusion 3']: + from modules.model_sd3 import load_sd3 + shared.log.debug(f'Load {op}: model="Stable Diffusion 3" variant=medium') + shared.opts.scheduler = 'Default' + sd_model = load_sd3(cache_dir=shared.opts.diffusers_dir, config=diffusers_load_config.get('config', None)) + elif model_type in ['Meissonic']: # forced pipeline + from modules.model_meissonic import load_meissonic + sd_model = load_meissonic(checkpoint_info, diffusers_load_config) except Exception as e: shared.log.error(f'Load {op}: path="{checkpoint_info.path}" {e}') if debug_load: errors.display(e, 'Load') return - elif model_type in ['InstaFlow']: # forced pipeline - try: - pipeline = diffusers.utils.get_class_from_dynamic_module('instaflow_one_step', module_file='pipeline.py') - sd_model = pipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) - except Exception as e: - shared.log.error(f'Load {op}: path="{checkpoint_info.path}" {e}') - if debug_load: - errors.display(e, 'Load') - return - elif model_type in ['SegMoE']: # forced pipeline - try: - from modules.segmoe.segmoe_model import SegMoEPipeline - sd_model = SegMoEPipeline(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) - sd_model = sd_model.pipe # segmoe pipe does its stuff in __init__ and __call__ is the original pipeline - except Exception as e: - shared.log.error(f'Load {op}: path="{checkpoint_info.path}" {e}') - if debug_load: - errors.display(e, 'Load') - return - elif model_type in ['PixArt-Sigma']: # forced pipeline - try: - from modules.model_pixart import load_pixart - sd_model = load_pixart(checkpoint_info, diffusers_load_config) - except Exception as e: - shared.log.error(f'Load {op}: path="{checkpoint_info.path}" {e}') - if debug_load: - errors.display(e, 'Load') - return - elif model_type in ['Lumina-Next']: # forced pipeline - try: - from modules.model_lumina import load_lumina - sd_model = load_lumina(checkpoint_info, diffusers_load_config) - except Exception as e: - shared.log.error(f'Load {op}: path="{checkpoint_info.path}" {e}') - if debug_load: - errors.display(e, 'Load') - return - elif model_type in ['Kolors']: # forced pipeline - try: - from modules.model_kolors import load_kolors - sd_model = load_kolors(checkpoint_info, diffusers_load_config) - except Exception as e: - shared.log.error(f'Load {op}: path="{checkpoint_info.path}" {e}') - if debug_load: - errors.display(e, 'Load') - return - elif model_type in ['AuraFlow']: # forced pipeline - try: - from modules.model_auraflow import load_auraflow - sd_model = load_auraflow(checkpoint_info, diffusers_load_config) - except Exception as e: - shared.log.error(f'Load {op}: path="{checkpoint_info.path}" {e}') - if debug_load: - errors.display(e, 'Load') - return - elif model_type in ['FLUX']: - try: - from modules.model_flux import load_flux - sd_model = load_flux(checkpoint_info, diffusers_load_config) - except Exception as e: - shared.log.error(f'Load {op}: path="{checkpoint_info.path}" {e}') - if debug_load: - errors.display(e, 'Load') - return - elif model_type in ['Stable Diffusion 3']: - try: - from modules.model_sd3 import load_sd3 - shared.log.debug(f'Load {op}: model="Stable Diffusion 3" variant=medium') - shared.opts.scheduler = 'Default' - sd_model = load_sd3(cache_dir=shared.opts.diffusers_dir, config=diffusers_load_config.get('config', None)) - except Exception as e: - shared.log.error(f'Load {op}: path="{checkpoint_info.path}" {e}') - if debug_load: - errors.display(e, 'Load') - return - elif model_type is not None and pipeline is not None and 'ONNX' in model_type: # forced pipeline - try: - sd_model = pipeline.from_pretrained(checkpoint_info.path) - except Exception as e: - shared.log.error(f'Load {op}: type=ONNX path="{checkpoint_info.path}" {e}') - if debug_load: - errors.display(e, 'Load') - return - else: - err1, err2, err3 = None, None, None - if os.path.exists(checkpoint_info.path) and os.path.isdir(checkpoint_info.path): - if os.path.exists(os.path.join(checkpoint_info.path, 'unet', 'diffusion_pytorch_model.bin')): - shared.log.debug(f'Load {op}: type=pickle') - diffusers_load_config['use_safetensors'] = False - if debug_load: - shared.log.debug(f'Load {op}: args={diffusers_load_config}') - try: # 1 - autopipeline, best choice but not all pipelines are available + + if sd_model is None: + if model_type is not None and pipeline is not None and 'ONNX' in model_type: # forced pipeline try: - sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) - sd_model.model_type = sd_model.__class__.__name__ - except ValueError as e: - if 'no variant default' in str(e): - shared.log.warning(f'Load {op}: variant={diffusers_load_config["variant"]} model="{checkpoint_info.path}" using default variant') - diffusers_load_config.pop('variant', None) - sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) - sd_model.model_type = sd_model.__class__.__name__ - elif 'safetensors found in directory' in str(err1): - shared.log.warning(f'Load {op}: type=pickle') + sd_model = pipeline.from_pretrained(checkpoint_info.path) + except Exception as e: + shared.log.error(f'Load {op}: type=ONNX path="{checkpoint_info.path}" {e}') + if debug_load: + errors.display(e, 'Load') + return + else: + err1, err2, err3 = None, None, None + if os.path.exists(checkpoint_info.path) and os.path.isdir(checkpoint_info.path): + if os.path.exists(os.path.join(checkpoint_info.path, 'unet', 'diffusion_pytorch_model.bin')): + shared.log.debug(f'Load {op}: type=pickle') diffusers_load_config['use_safetensors'] = False + if debug_load: + shared.log.debug(f'Load {op}: args={diffusers_load_config}') + try: # 1 - autopipeline, best choice but not all pipelines are available + try: sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) sd_model.model_type = sd_model.__class__.__name__ - else: - raise ValueError from e # reraise - except Exception as e: - err1 = e - if debug_load: - errors.display(e, 'Load AutoPipeline') - # shared.log.error(f'AutoPipeline: {e}') - try: # 2 - diffusion pipeline, works for most non-linked pipelines - if err1 is not None: - sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) - sd_model.model_type = sd_model.__class__.__name__ - except Exception as e: - err2 = e - if debug_load: - errors.display(e, "Load DiffusionPipeline") - # shared.log.error(f'DiffusionPipeline: {e}') - try: # 3 - try basic pipeline just in case - if err2 is not None: - sd_model = diffusers.StableDiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) - sd_model.model_type = sd_model.__class__.__name__ - except Exception as e: - err3 = e # ignore last error - shared.log.error(f"StableDiffusionPipeline: {e}") - if debug_load: - errors.display(e, "Load StableDiffusionPipeline") - if err3 is not None: - shared.log.error(f'Load {op}: {checkpoint_info.path} auto={err1} diffusion={err2}') - return + except ValueError as e: + if 'no variant default' in str(e): + shared.log.warning(f'Load {op}: variant={diffusers_load_config["variant"]} model="{checkpoint_info.path}" using default variant') + diffusers_load_config.pop('variant', None) + sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + sd_model.model_type = sd_model.__class__.__name__ + elif 'safetensors found in directory' in str(err1): + shared.log.warning(f'Load {op}: type=pickle') + diffusers_load_config['use_safetensors'] = False + sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + sd_model.model_type = sd_model.__class__.__name__ + else: + raise ValueError from e # reraise + except Exception as e: + err1 = e + if debug_load: + errors.display(e, 'Load AutoPipeline') + # shared.log.error(f'AutoPipeline: {e}') + try: # 2 - diffusion pipeline, works for most non-linked pipelines + if err1 is not None: + sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + sd_model.model_type = sd_model.__class__.__name__ + except Exception as e: + err2 = e + if debug_load: + errors.display(e, "Load DiffusionPipeline") + # shared.log.error(f'DiffusionPipeline: {e}') + try: # 3 - try basic pipeline just in case + if err2 is not None: + sd_model = diffusers.StableDiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + sd_model.model_type = sd_model.__class__.__name__ + except Exception as e: + err3 = e # ignore last error + shared.log.error(f"StableDiffusionPipeline: {e}") + if debug_load: + errors.display(e, "Load StableDiffusionPipeline") + if err3 is not None: + shared.log.error(f'Load {op}: {checkpoint_info.path} auto={err1} diffusion={err2}') + return + elif os.path.isfile(checkpoint_info.path) and checkpoint_info.path.lower().endswith('.safetensors'): diffusers_load_config["local_files_only"] = diffusers_version < 28 # must be true for old diffusers, otherwise false but we override config for sd15/sdxl diffusers_load_config["extract_ema"] = shared.opts.diffusers_extract_ema @@ -1297,7 +1262,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No errors.display(e, f'loading {op}={checkpoint_info.path} pipeline={shared.opts.diffusers_pipeline}/{sd_model.__class__.__name__}') return else: - shared.log.error(f'Load {op}: path="{checkpoint_info.path}" failed') + shared.log.error(f'Load {op}: path="{checkpoint_info.path}" not found') return if "StableDiffusion" in sd_model.__class__.__name__: @@ -1981,7 +1946,11 @@ def remove_token_merging(sd_model): def path_to_repo(fn: str = ''): - repo_id = fn.replace('\\', '/').split('/') + repo_id = fn.replace('\\', '/') + if 'models--' in repo_id: + repo_id = repo_id.split('models--')[-1] + repo_id = repo_id.split('/')[0] + repo_id = repo_id.split('/') repo_id = '/'.join(repo_id[-2:] if len(repo_id) > 1 else repo_id) repo_id = repo_id.replace('models--', '').replace('--', '/') return repo_id diff --git a/wiki b/wiki index a3d7ec999..b445dda53 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit a3d7ec999fafc05d75dc5d0ae564f02bc689786f +Subproject commit b445dda532e2c0a1ffa4ed01451bd33a11a06658