From 98afdd0a8ae3a6393f281eff7321b455edc1496d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 11 Jul 2025 13:19:29 -0400 Subject: [PATCH] add freepix f-lite Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 5 +- html/reference.json | 8 + models/Reference/Freepik--F-Lite.jpg | Bin 0 -> 52266 bytes modules/modeldata.py | 2 + modules/sd_detect.py | 2 + modules/sd_models.py | 6 +- modules/shared_items.py | 1 + pipelines/f_lite/__init__.py | 5 + pipelines/f_lite/f_lite.model.py | 455 +++++++++++++++++++++++++++ pipelines/f_lite/model.py | 455 +++++++++++++++++++++++++++ pipelines/f_lite/pipeline.py | 303 ++++++++++++++++++ pipelines/model_cosmos.py | 5 - pipelines/model_flite.py | 63 ++++ 13 files changed, 1303 insertions(+), 7 deletions(-) create mode 100644 models/Reference/Freepik--F-Lite.jpg create mode 100644 pipelines/f_lite/__init__.py create mode 100644 pipelines/f_lite/f_lite.model.py create mode 100644 pipelines/f_lite/model.py create mode 100644 pipelines/f_lite/pipeline.py create mode 100644 pipelines/model_flite.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 23f79a8d4..b47fc71f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Highlights for 2025-07-11 In this release we finally break with legacy with the removal of the original [A1111](https://github.com/AUTOMATIC1111/stable-diffusion-webui/) codebase which has not been maintained for a while now -This plus major cleanup of codebase and external dependencies resulted in ~53k LoC (*lines-of-code*) reduction and spread over [~680 files](https://github.com/vladmandic/sdnext/pull/4017)! +This plus major cleanup of codebase and external dependencies resulted in ~53k LoC (*lines-of-code*) reduction and spread over [~720 files](https://github.com/vladmandic/sdnext/pull/4017)! We also switched project license to [Apache-2.0](https://github.com/vladmandic/sdnext/blob/dev/LICENSE.txt) which means that SD.Next is now fully compatible with commercial and non-commercial use and redistribution regardless of modifications! @@ -35,6 +35,9 @@ Although upgrades and existing installations are tested and should work fine! *relighting*: automatic background replacement with reglighting so source image fits desired background with optional composite blending available in *img2img or control -> scripts* + - [FreePix F-Lite](https://huggingface.co/Freepik/F-Lite) + F-Lite is a 10B model trained exclusively on copyright-safe and SFW content, trained on internal dataset comprising approximately 80 million copyright-safe images + available via *networks -> models -> reference* - add **FLUX.1-Kontext-Dev** inpaint workflow - support **FLUX.1** all-in-one safetensors - support **TAESD** preview and remote VAE for **HunyuanDit** diff --git a/html/reference.json b/html/reference.json index cfb4a7dcc..ea94acf50 100644 --- a/html/reference.json +++ b/html/reference.json @@ -189,6 +189,14 @@ "extras": "sampler: Default, cfg_scale: 3.5" }, + "Freepik F-Lite": { + "path": "Freepik/F-Lite", + "preview": "Freepik--F-Lite.jpg", + "desc": "F Lite is a 10B parameter diffusion model created by Freepik and Fal, trained exclusively on copyright-safe and SFW content. The model was trained on Freepik's internal dataset comprising approximately 80 million copyright-safe images, making it the first publicly available model of this scale trained exclusively on legally compliant and SFW content.", + "skip": true, + "extras": "sampler: Default, cfg_scale: 3.5" + }, + "NVLabs Sana 1.5 1.6B 1k": { "path": "Efficient-Large-Model/SANA1.5_1.6B_1024px_diffusers", "desc": "Sana is an efficient model with scaling of training-time and inference time techniques. SANA-1.5 delivers: efficient model growth from 1.6B Sana-1.0 model to 4.8B, achieving similar or better performance than training from scratch and saving 60% training cost; efficient model depth pruning, slimming any model size as you want; powerful VLM selection based inference scaling, smaller model+inference scaling > larger model.", diff --git a/models/Reference/Freepik--F-Lite.jpg b/models/Reference/Freepik--F-Lite.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9343155459cfe1561c933a916a1175a4ab79eddc GIT binary patch literal 52266 zcmbTdWmFtN6E?ayL4$^cMS^>977q|SxG#|4wzx}x0KwheT^53~NO0HS4#C|eXg=Qe zvvcmRdvDFDnLgb$)AdYuS6lVJg?}4>xAM|*(f|Ym1c2e|1^D*|U@7HcWexx+Dgu}R z0024w8G#gl@)|;TWl;pO{}Yx*U<4rjHy-gdkrVKG2jIQ3(kuUm3}5*_@BH`aU}R5RsIUm6K)RQ`S_H*7ztREA^j`AYfr*Mk_5Ydv zbpr6v5nuI3Lcj+g;vpd6A^htFP`+x9^7>!?^K1SWAs`|lqoAUpzrnzK6=--1Ktw=7 zLPSPFK|w}-mG*f}2O#62;J@b-M=f_@gZLc@ zkd%&|fsu)shnJ5ZEFdWPNlIEqR!&}BLsLszM;BsZYG!U>X=Ux~;_Bw^;pr9lBPci| zG%P$mAu%aAB{eNQzo4+FxTLhKytb~sp|PnM-qPLE+t)uZI5a#pJu^Euzp%LUXLDiXvP?(hA>e{dnZ;{4zAKY{%}aN)h;LPSPJLPq-!E(ApPS4P4^MtRSP ziZ8B;X5>gf&E=0yC=r)e)AfdiTkV|4*y%S0F)hy@x{Lon`!BNpcfbPve)x@!fW9n;Q_<|D`8!KIv-}vWl*lo6$G@T(9p=sSMv7hfYqGw%ZR!DvDU6~Mqsny znYCrEdhh%n>E3v;0@twI;lC;zqDi7lvejn9&c1SJB)csLB_S!`GtHaOqB2TtcaBIj zd+H`T8^rfZjKS1gCP}DqV zuFo<1)@h7GaCIy*1o0zZj$~uS-{9G44KB&7yKt-zfLWYS?dvd1Zl z^9nBbR2lA_4*%q=BKehrY75s(eK;g}(OG>o$V=6s=IhWHg$Yt0X{aPIQ|)JIvg79DunBu$F%;hRFMqr8h7tPPZg~_+qg2d!B zSwCk!Y|Xw?nlWNOyEFd?09l}&mr-h}mwoi0JLQWRA^L?&sa7QQ#yXAH62dP%6c#&k z_~$S0Dac&3+0i_*=ll9p6*TVslo_psIcY3~YZ;1e2om7=&$tr$OF-`I*O7j(&dX(t zdf>AqvRQBc@CJD8;<75&f}zTKllDdFI+A2DY(hB4dWsA~>cC9a;{LCSf7tbI5vNZP zQJiX_2kN$SWc4@i*+zL8#a#Xrqi-dre{`O?$ISP&14QWm(9_DQ^m5OJ9;mWG*ac}S zf=r>ZW?{j$tF!);p3)vglSPk88BVhO1|3-8;}EX75@5L%$4L1Qp+3v^{hq(>SaZE@ zV}eC1ma-u>b|L_Bs2LmUXr^n6fPyWb38SuVTN9Wyu%q4OVU%-WMGl>r=(gs*rb)zV zl{dF?dIpxuWtAS2sM~07xF~G9gl)149fC3@Q80Wv|BA&Wf(H8(5w`!%J(r2r_PIZ*WFSFIRe%1m3{W6Qs6?(^OGw={|GoS1R-?0-Jj$<=VqOx22 z*W~W~x4CVJL(?O|FVvZ3+g(Dz7@sT>^lx@fGbX=WG|q)PsP-^b*(+VViNd4Z^EL#{ zlU6vxy2zOtJ$zle5aDM89#I8NdYF8@Q&_FIe<-&!M;`(Lyx)}ng|lv3+J>Z29romk zCJlsYVAV{_)g?ZXb;md|YMCgHAkssKjIms8eYCa2tbI z#*a@&IH3}M3r3|gM%V`oRf(xyNa}OTl%QZOpjvGB64_Y}iDzlw-XxeI*nw%wNU9L^ zQeqjF&bGA1WfI$7TvLShp~v&;@c2Dec@AbzWOt%5vLkPA@D{ZH3S@>O@jIBN2KhaEWf3#FH1|7g<@GNfyJ`m*W{ zyHCf3A|zzBP&y9_+MODk@O8%Nzi+#^h>&PLN{?Hq>@1vB-uSyZ8HLS$*H6e!>b3qn zDV?63K3XHt;ml{pxS^>O`=uCFaK{g`Da*haRv6cs5Xc-du?E; z?M4!z6**VT!>uxlJu1ClSRA?1zz}JX&S$ny}3NW(fe=ZCh7wZhw@N)j!{C*(huh*c-kTDj+if($~0@-H^!3);w2@ zfAQ5qu2L59+?s0m+|`#3L*O-rHs8k3GsI?h0>lKXya@&uWjo~#bRWh`Tut91#tr@h zaHtNco-O5Rx53re{w!(R%ovEu0GInzn=ZH%-}Wl+nZ#vtzYy*Tzyb(2@1$-T(HBIR zj=Vwp=vXDcp#19a{GtV|Pj7-ik2C59WeA!MGGD#=G_()x4q9E_TgvfZeisaH3fox( zR&pgDD}Nv_2Yz^KmK9LCov@V20=Em$aV07smW{y4)YO<~EmC_6H(Dn5BVCU4b@4JeX^q^=dr5-K|NWI1E~A;X6{m4s_n54vytc&?4vAk8 zX)jdd9StK?n9bb4=)0(YyYkn~wAo;|vJ$~IH`ZgQvRp$&+h!`$Gb@7b^T}%egmck){-X;rRy$o_o)3&rS%JB2y9596ETwxN%4|Qz~$8k8~vZK;|xF`Nwc2 zT+V(|Zq)3Tih$A4!57##U?#yW!=V7Bt@dMi{2}pWh4kU0>>(p?S1{i`1-o-x=i!F( zkK(L4T>+4;);T+KH%Y{TA3~NaQ@NKWXlz8d zV_w>MmfWRyUCo_aM){6Biw1*|Xx9nTwm{)cRl4>~>=kg~bzT4{4?LWsl;qw@KPQ?0 zeF@S9a;Q<}w>jRU(~IRi#1Sv8B(~K%Nrtgh{*K`j=EoZ5p4evMHy8Q6@YkDq_(bq6 z{280OXDT3tT~Js&WRppjTD2kV&u{iGKu-8~L!37NC$Rz097TyUN_M6i$zZzl!q~i_ zz@EU-f~G4zr|YwXUK*A|A>ReS0eI1T!%(GYI0j`%1eIxJwYY;wQ>bU;#jjYnWX+R; z`H9X>Aaf8GN1C@>3!O9#Qe%Wb65K{E>CdH@6srOiPW&VIRNeip+6QW52j8z>yzr(1TuS^Ctx6O-& zN6XZsqWM2}BpKN3NVAV6zyC=vO}RJKusap-#k0H>K{#-M((Bt$>SaAtM_-be6~0Wh z0Vk8YHS*WIp3%)LsG-vcSoI@#_7j}TTs7a?5Rr2IBEuc2I8n=A#APdC&pHGJIrnA#GRAlCd1hkbV;~F@?O#ihc~tXAEEQTW zV+B*n`<_=ubePzStH_iMHx_uY4C8Pqt6t|{k|%Gkn>?sf&%s_!hGcVuv2KezQSyyE z`V@;3$SStDQNY(-(e2*yyGE*W_w(Wr^lLX=+369|30=t zd7;Y4tp_S0j%P*WeS>V@+8y_M0ff+j+!D@(j1cus!ZQb2eiU5|kX!j_#|II=BvO<` zzD@HX1}euBRq4Ji;cdr{#Kb4tPPE&I;^k*aqQ?IKok66ig*&%$4RH0zX^&(MG}XyF zLVXmYX38B!Y1$KE@V<{F0=4#)ME*P4bHEUcoOH+573!I3Bx!4JbD_?}QxzVr7-}5L zh@U4T{JjTQHKAD>tWvd)A-19MsD)SU7e)$2b8&jxOfF^hD^`vksM)cY5Fkc6E1z*_ zSs_|liB1z<%ahUhMrVr#_z$2XP+spH(r6GgU5(Pv73@Gdx3v?etmHil)GzQ<|MYuk ze|;*4?RVtPyYhD`=O<%D%jk)61mwS)Urt!Rks zCJQF-DnBV1U>HL>wi{l>ot&4SHaE7Gbmu`=V*;8m(0S1C#-?yB;gR?HeJ~b$L9IH* z*4%#U!H`$}dsT+kG(DymXkl1C9(zPN;6J)emk>f9DOalnFgdf8@>o15_%WYP78Ta*I zqFRkOE_eL(&-i;MmRE>1Z+)#t6MFe019W!jxTCXh+bPb!(60ukl^ryRwIq5{_9B_` z|2fhTJC9rS&*6B{@ZN00i8?tunbRej`h2<+DDv#t&yqv>bcwL5_+3P3x-H;>owxEX zsHim-sI$es?!M2$tk>8a`|D4_n23~%%PrsSZSo1xav#`EB3ATk-jRNuld;p;m4DKN zpS6}xPnw(Hl_Qj-_cI|*=R9d)T)bXfdJ37HTH1WZU-2azMaSB~K`yf;kuhb)`|*55 zIp@IdED6wh&ND5P2j_mqTJiDmV2fy*w8Gx6s}qnN3OQ4Aw)Ttld<40MNr9+?Q`;@d z{cXnsov$sHD`^7anIDMQV$zbU*mu$}hEjkaJ1X3scJ=sEqA18)1qxAk>X;zQn-h864BJ;kxj;(Dbl!gc3dg3)H^M6{B6brUyX!ms!MK(-u{(2w`+m| z+7lPGi=xRGS;Lj@m?rm{Ur9gRL2L)b@6`RTciRqcG{1dff+d%HAVjEDq$Yg&D@@8AN6T#tsbdZ8#{Utz^=e+f%f!YFJwOI4MmZb(@yx zbP#DP+7|DJp!>P9#x!w;GPr5s+5WK zQeiAJExVH^14%JW&g@XMu7j^rFVf#w+{;pXQ!6B8CbsWNB3_gSY7oH=M14;|y8wwFtYkU8;kg^@i_ ztQk%!WzIDZJbdR=r&ee4yS_eG0!g>&Mc1p4d}rI-mxEp=q#fVGm_2o9U}^Gz>z|4p zg(kT|l34dcXvQS5TQYCD?8dkb8cK^F^SBjt0UeJUq&;E4W*(1yKk?Lz{NscwgxH@i7=NdB>P;6)&wjOmS2?- z&7Dh@b~khXn8}E=GD9Hzdx#MvU5=v&P$Z+S(KOV-anxj8QRMk$oK@uUbQDPO58&3W zRLOOhW+$89JdiTZ!xskZi9ouT6Ss2eR79jJ zxUz&ux#(xJ|R2()SJ0zGw{4tkM7Hu`>Q^;dO1xdILa3{8ZaRXo0zNfT8 zyYahGxxN%|h2OtRa)M$u_Y%dh;?6INfj-E%Qv3_~r0n3x6Ya%y1(vb)@OP4EN68Fv zV@jrMwrvD$1)C}5W`j1*b*;6D#&9bJ+Yz<1wNWXWYvjih%%{BolBAC;+@i-bf;V<9? zkU*aVea!&hxZj2*dtvH~8aw;XxItPC!=pPdN6mulAtm3@k<%?+9Oa+Kp{K5B@K`t{ zrvR%b?QCLmJ;t&$jz`h>y!ayU#c^B-x2D43ImCg+eM>TdJluPmS)z<;s|hlXiwf=} zFZX;3=y!^!EfTeb-PNttKMmFliDfC*&l@caFrLvcB4>vXRaJi9cpEquV@b^AJH%Y7 zsiy3;NYA?>j(Q>lk|vreP4C3k`u++To)EHx)m^;uIU4*vi@S_Zy!K*^vcJHIsQnrLxKh(b1^n35Eiv0?0*dcQcoxY1WKDe_jx*zzU1 z@wv#NW?3#t9Xr<4CrQL-a@E``i>3|yQTdzqRo3H%JXo2-Cd+@j+T+8E7FRAiRB zk;0b7Z0RsoJ?2Sh8BeJ3URq!T>mMLTK1O=(tNM=-n+%$(#(;kSkF4g`7FCn9MbctJ zUhR=3m2?x8^Q6GKqZbiMuxs$dF!StXz>3pWQ1<4O*~}eHXc_r?H>T6cjnJ^1J{r<&dthKlv!Wf-1Ca% zY=NgEaaKKLxVYVmS&f?oa66c4+lNKLQheknoeOcWxA2EQq4lE*>bB`tgXs~oZMyGEcCn|iuURI<`;_bPsmLbL zG|$)>*O;ktezvFC752IMJ znoMoQEuCV8-!4-U2d%@GM8P{G<1$C`_`CEP_paP7V6kM~nRWIfoWhr$0#8s^;%Lu_tV7oXk9+|J%e5hT ztPPevCgeTJU~etg^84E7a#?p-y*qE2HR0_f*vw*JfS~sYh3qj+d((=^Rxl-3!c)J; z81!HYEg8QTZkem%li64xGC1hUsfn`%F6HX>bVOWk6Mt$ zKBPB)ID6_-h_v}!M0jkkue&_)HW{n8(C%IdVN5frFrFKV`fghhE{x!xJ_c00W{9%= zJsap8jpH?E(zLkCsgO0&W742`dTKIeJag+fOlyU@agzDL+QprTfYC3_D+WRatqQ6T z8I;b#;_DHvjOx0+EvvS`W%kLMslYl$%d7YXTkD2cnFNJ9^=`zk)=jG8e++z^XL1N_ zN&+>p4bU%yrJPv=2w_xFc`BKiTob}2Q?q^Kls?IFdHX9xBa7D~OhCTnxoP9Cv}~R3 zwd<1+_)p>#S0teO4h6lV-=b?K(&Gh0YxYZ8G}zHcS-At_m()~1d_trU+@#D_2#r%=_<>Ro zY15iOQLa;6BGi~WFkwMknJ>AtS!3viFko~C0#u3igRJpkzFXVzepEG_>0V{_PadRh zESxflbC@f^A*-;(pg$Y z`DDJoNN)Q)cbCv(W{&gRX)ob`=H#p#^%`wz0OZs}4T64ff-%wUV2F+_dI6OEByI z0hl|R=8}C7`w#TkFaiGngY$zLMWXI#@Wa01Dz|rylh(w(pF6qYCg(`4JK$*Us%j;? zMS+Pv?NPt7=-L}pyzD+?)xIBF+IxEpIkoXLhoX_=ezf&e1ATjEq>*5 z=(q7M|HTQfgnq&N)%Auply9Y%U=TsJqScRTNPT8;`)`dB5;lGz$`HNn^JHgu-)I3t z2QOL=^zYXBL>y;t_O}q)hxF@sa|Q)#2oP~lCKY!qA#*l|O1NhH2RB0ST`@ITJKtR2 z&dg@-r;;Cr!^2#$X%okod*O%` z#elm(YYf`+)>3UlmcxOpVUyuJ7ozX^J*wc;&uM{(zw5`p^oP5lR&L-km4my8vWN($ z`!+_McmDv%-<|LMRW>vDqNJ133|x;Yr15Ja2Ltiys+qlfd6{aG^*$cBln-)T90Mn| zba}EP*VwgM%=}hX5m-X3WNCkr`#7sYx@^qp4?XoQKzyiv7|P=N59v$#5f*LK zj~A^DJ?+}rr??3)zaD6yBiMiGM>xaoyzB zJMqtPjXSX%0wQ99DuJz_Ll)o*`)ph;@nB~e8dyYGc?xbqo1)D;j!UubzJ8=9%`8Zh zHCB_H?4M=^Hv%y`2hPS6S9uZ48nL_;7EZqgBr0!IBO(Gf!Wev#LA^b z9t`|%SWz2%4l*5%s!TeCRdx`i3hI=YyXe24GmShS9@Jg##~OMR>)GFt#ZY}6*Li!Ar@y-_a2T1q6P?*nDzqiB#*u^4ykRvDml&b7Z~v+3p(0{kH%1E=8V>Mc z>a0qVcAHIo)U$zr@l)ISRs=V8YYHQ=mow0WL37#NBTnsghxWAsACZDzryAy^i}hpp z)vOK1-C;sJb(-p?$8(4c7EA5~MhRJY~nQ3=r-h4CNsmzFz^Z!ZbzEvm$c6u;$SV#br) zXr8GRA!kBJErc8Eq{}BTsVTlk`X(OiTG889={omNG_0CB#{}g{*^2uOl5*|Apg{kA=;Tg6dGe#3q_JeuZ?*D9+QCN0OS}^esMmiKI)hT`;x`Db1@U{dn;8pIC`RY3a zE@152J_!|6RPGX7UWctDJxP78;e=R2^4JGAtE(CnAItdR6EtyCEfMS32Z}U9q$9*K z2xL2GGv@Wd)U08XaXp^ZWWh=#*IT^5>60uhO}RMJmV;Ii(8)oM4mGj=?Bh^>a)D)&W zef==v&XZp_wiP6?+%gzUQJEwR)a*@PAhCHgb&hWEQ1Tyn_SH*Msb z=5sB{)XlCls|C!FXs6Lm03>8@^Y}H9$bnxgd+of0{HoV{*9$bdMByc=mE&Xi`+#(g;P^>uvNdmSLX@1(x?mqXQoYsyEy;`KS6Pi|E{Nr$efVFoaq&I z>EEMMe8R%A_QVoDgu;Dr)-Dm_yJPdSYPhPV3Z8CZzSTpO?xbZtBaW-s+DdZh7*p(I z(`z;Lk7@NI6F&!$0|XFvtgYr?Z!TxFzlQF~V}GG<^OCiczE@jY^m-@kuX%QDX=>p< zs`BB28~@EUVnc;s)0CXAxnrbPLH3-z+K9@dXp}>Tg^jV~5O!c&j9r$9H5p9Ik`6!? zx2-YnG!l<+U6ooyWa`{Hv@(BQ86N@o^UIOO zgqOmfD!gY~jQzGM-S5qh(?WORmSw#l0%?s8cY0@(UOkGnPLPan`~;y#asyWucMXH_ zg2+4YfsP>IXeluoN}vT%Vt#UeO@c`S9A)gJ_1o}88H%z=Kh#fi>d$QD0w<^ZNKiUE zP`R`$;^%giV@q%~w0ODA32e10s?)WXy*H^%RAt4##CsjLP9v(Mb%bVye`^JW+Add%Cm+E)S-5tR@AED`hCp71d70u878}{o%a*+12Da@F=#WX(p6d z>py>bvt<3jXyFu(%@tE>21Xvvalb{bai^}YlvcK?Gk9PxB-4>CfE?Yc-r^m_Pac(j z!P?8}?Z1UuTR5|8&OSt$)#^wK^3|ve-?Naju{~j1+6koj@IK-kO@8M`6KP^Me_WQQ zPhb!Uc@Qc7GV7sW(KyVxmi#p0_s6p%wZ%%UuN*_H9And>onD%HaQu8qjoRYc27!% z@kZt7Qdn^}d=yQ3F1M(j#UE$9!8OixWesyCEza1Z(S*fQz{XcUZooCe68sO9V*p3b z%J1WRG0X8;n63SBRGT{&Z5aFc)c4h&GUP4VG~|oY_tFyCLMKnm1voMu3G~G<8t3D8 z=`P*UOGXT2Gds#+IeV(hU&tb~BJnra9TtV;N-MPF=f%I<{vZipt-R1*|QTe;9#4KKd{e*~Ok#d>)gR~L=Nch%%NjZm(!B;>#x zXPkDY{sHnFcWUs*i^qu1yazj6pP2*fE-^1vQFlqkS!8fLG1C|q-+!nAzMIfOac(M6 z*VX=fg|9pABjl^Os(_PRxtF-SV0stB{q0mI`ulCUQorRL@2CY}gKEB&a(4q9hLvnLdYWrBj@48EVZIDKvzEcLDiypXg zM4v|PmkW?}eN^2g?Q}P1q&Y|?qp2qED>-!Ihely1&6 zsuQm(NIcRFImfr2F|ToaTD_h>PDbR{d@^Tz`+{qw814~SV$pGDh;8#YNu*ofrvoN7 zarwT5sc?W~O@3fhibT(Y7{=&T~4&(QHCcEx-brYpOllK9DuR@vzv zV54BP^GZpc__4iMjlFE^t<#e{w$+r9W?dk%lYA-(Z6?5{yy_%gP?2~SEncr(qj0AJ z+Z|;3sacX$T7%|dqFr`l#o%M9EV3cevo)Q014_Z9^}+On`!B>ei5{w&KfVo(c6-yX z@5$h8DHL@iWJ*6(b>Nk*r16T@rGSr^cuRYVl{yCQuS-P0E$rUjG$Atm21K{gCC}Jk zVpXyq$JxY2w~WOzHc#>0jYxq`+j%Sd`(0%gW7C;XdU^a;jGFwWgu%}GAK}A3%7%dE ztzy|Sm$@Ora?#4gN3R~fTdIz`D5pAW&eokm+8l9(g{}&@kj8i~nyTiGu!6PPWE+<% zp+`9{>Q^tlU=;IG^%%7^YFiuV%3&Rx#&e*^=oy|vXAAFzy$(R~3sOi;xT9sW% zrms`nnF*zNTYfe3fxiB>BA@%#9u6ZcKe^(dRMmmfjE?$e%Ur8>mb!kZag1u>u`^JO z8E-zyoJDB(!b#177gzs5;9;_&oG9l z9=uu7B72f}qRg9$1tbICb>_{*&N9Xo#`}%%Ou)hARPv_u&O81CdLNw%31;e?AGE$r`N&VEJ0-8k>X;NuOKDv3bH)`p$$?d;B7Rls zuSUEDR%M22I-n{fZ(sy0c%JPP);_JEL|)xftXd}$U@OEAG;R%B;K(S3z!+{759JG$ zDholT8v!z2w4c)Jz=I{`vC;&JzksZptHuZ`OPQlz6o zdFN26U^4rYT_;0>OOI^qh@fu~^+&CbwFuK}!C#BaNc2dgXtSL>x+=T^YWLl|q)zg6 zmHFIc$Pl+Y)+ZbUbGm=R;z_!9>=9^k(f|xkEKE{O+tx+ODVk*mOYC*F6_p*P&OC93 zqr-)+f$0r*Z+tNV&VS6R&5DXO2L=C z8grUvtyM_E?iciO06b`ojU*3_5AVsbXWKL!Wtn^C>I~vlDs|S;qsnibJ5TyC4UMGkysNfW$0jHt(}sZbz>6|IsGO%cT~UcM~{k#~)InGnUemoAS$>00G=_*N@g z$)v^iqJ5Jzj2I7>jgztmtN-Yg@dAkzqQ)4%wF++R0nyf}X2V=rR6W@9hKC)hQ~o4+ zzpgA!IXHcrLm#3udSSW1Zg0pMRULWXdO4g6LZ0|G&#;NU`K#44-yHjW{~HtyKf!W` zMi_FYZ60klwI}PQ#5z|lN^3%N;k*c40t>sYuJ`~e3e@-mMQ_52a5a3}{7L5TtUUwr zck!X)9z_3D^1A?5*jc4213S`aPXLO>c^yeTEca@c-)k-XTZxD?JbLkoqZ|lh*R7f4 zLobk>qiJm9;1pVHeeLlwb|vIltBVXmtAo|8q?Zn{b>~jB#{YcS9A6y7Nj*4|nu)F4 z9G?*+k?y+O3h$k6AaE>!f5Bktx&c>0Y8pYWU+JoxPYMRg%^#s1L6J@*Rv`=_4I{F-(+8AH4kb6Qc%=tQNc(!MQTok=om;8S1>^idcbYkSzBP;HBaB z&T-#TRx$RtUvRIi0tMJ3>)ACxj$#nx7~5ORhj~mgM5P(wQ$|o5K+DcYEbPCQE1OX=G7~Bn}vVCfq z`xI|)PbcjKBDlkyitt`LoTNR)BM$!)tW$$TyW{bo#xgv*>&8a?lqMQbUxInqWvA6H z<+|Ky(n+7)v~;-U4@{w12I zZLg&^AXs2v5~m@P!ZTwmYe`O7TBN;<=j04!_V&XUh)B{%|M5h0REO}a-07Uo4`?Ry z_G8l(53f%!-xkC(dIW3|ig3k=67vrLDfKfwbUL?9=OqUWP{Ld<#}Y9YKGk{OR&5##qO?~m=+eh7zS12o z-iC~-e+!tG=Q$d&;)7K%`_BLh4wM%$)BBN=`Rl}nn58Jcoy%;nSnDa2PDVgI5s<26 z#?l)U*#mYBM%69nIx{K;n58xH5UaaN4%L#cP8t#J#9v3+5Red+n!)gIdD_T(BYBfu z5$987FF0M)J&B&~;JzmyXa2hKc_;t#x4{Zfo&9BsaWKdh+$XSfJ}Tc*3i%{UqZMMDNb^G=BgJd%z*L-7g((+{Mv2at~VVV41me!}FhDL2Jszquhw2N@&9T}JfdZ%1?I8m8hl?6e z!ohX%NFfybp63Alq2%w(<8xp-W7Bg9X%mT2W~^Amyy$Gsa|nze=Xmkm{I2pv{xpt@ z0so|2{8vCilEC#oi`XNQzOlR2)YkLP>wco|KY)>Q0sV_O?@^<7bRc=V_0t;hT(-IC zv>Aq%+58h!hhd})`_POL!#Q4zbcTXxUgoLC>>uD)$`e#ve*#M)s|4)`Leu@Q_l)$P z=!7Rr#Blk2RGk_a?q6x%i3ik?e*hnbzmq7cZar`!)4yL0#ICSYULKS$3Q=6(?v6u! z`Y}DEn@Cw)HaWKmPu6CO*gEBRHp=Q+xIs75uO~K!{LTtr6~+I_ryalZ{cA=8#19J^ ziOMt7@!xn=Ei%T4Fp`|qoFW4Xg87N3?pIi^YFH03^7KYG6#WCByN8BO+-h{(E%{w& zTqBXMyQ85tl68sF;sJ3I*t^!fUZX8hfn8gG7ozI_iYqI!e|yIV~twc(x`rz@}bTHx$VVI{b*BedIq?mPLB2ZDoct z&En7-ld4vGIN(0HQC+DuqB6qFWDY>jahWY`+b5PyTrf||r-5DU<`#?6SNGy{!%^~2 z^t17-_ey!e-sX*zUFlEZc=choDEs7s0S=;f?btz)BxyFVAK#f;Qt@HWy*Ib?4IO{b zFSy~JJYiuB!h(i=TeH};^#ZpHnWi<{=?66D{Gr{+w(NGzi*W9kuk8{N@W5?Wc`~60 ztZVRtwgH;|jTKwst5XAE*@(0)4~wxkVV3k-A(ao`3woKR};a^QakJ$QKp|5B^^9 z2tQNhc;~y8?@=_BFiZOv}Dad7vbm|-+`_*>ToBEsof=Y`*q3d z7v|x5&jRE8<7*!F2Ex|M3%eS84*B!0m}wZK;rz78iZ(&gjJh?i)3y>}@a~i)p69qS zfo=SB*Nd4Uok5E$TVRmtVXm!3y7>;?RkfLVD}p9O4!0@D=c#kJtS(#P788BRp#n_p zvx9pE@+oc?%1{=d<4*BL<+|Sir0%b?I`NL$xU?~RBW5H)RXI)m*0j|XTxnyBFwt(y zALKfv&k-euy`?Qtn#T|7rrc*)lTDBWUvqB!NE9Nobilomk zCm;U-Kp1w%hf#1fA9;7T1S_u!z0XC6pq$%wY^Xj92q0dTsJhsH{U~T7pwA!0-91-rnTDFUp7=nipZ#*4w z7Mo0EeGQeIV|KJ`>lb%wt?*t?F&hQAM*af?d8G&(Lzn&s0JuO$zdRRc3{C@p3n}eb zcCtxpFmxFzc^;yHXmzl8u=HTUqJh?UQdr5_<07T2jXM__=n;oZ)EGVarOJoS#!8V% z%BI6o1@@|15Z%h&Rw~16MK2=-QOP9W3U?KV($+^(%NZb5JD(LnXAk-^hW`L<&KL0R zYFwG3f@Qs!_O3qj#xu<()R9;oq%TUJY2;F)=5&|ZjlP_lr*6F!MoIb#^NYPwY3FGS zgtzyboP8-dx3Nl;R_9p$GmJ~MZE(HIwg~(ykdE%+d?{Z%kMDs~%_7twdLEUoL#Yxr zY#`$x=?sSJKaL$Y6b_=R@RWAstFe)3V#ZhGPw+)6=+Lq2?U|qY4Zmu zL_?n6tw>|RTr5T-BBbJr zj>AHcPt70{WKazNNYb~G$&-#arXRX#p*@6CLe&yVrjQB%lo~(+ti^?920Q)28bxA4tAR!=gyya=SESp3s}3P449axJZ%UbyHzYL2d1j8T6H$u&t4 zCu2$}M6oNcMj5IaJC?L6dLs}%l$x;O?u{+f+C>Uln~GpA>TV4J0@TtC050mx)%BZw z9u4yn(h>5;I-h@GQskE9D7UevxYwM|@ecJ}nIN}V)Q%fJ&W>b0MtUyeS-Kvv9=E5h z%xYtBcF7qz+J3#oG}9%;t<5dQj8nK-jU>~tK&(bdrx8WM#AKRrqTymPO+3+Xu@N-$ zMZ&~n(wtFoSY~NR6kIHF+Vg+KXW_N~0BU3Z0FKi&@A)45HonrI{B+Z}(mJft?V{p1 zPlz_padoE@84Dbl{MM)O)Urc$VdeKn&mQYoRc`9XO%sYojq<5(y&JW6RI)C-6>E^C@k#;*hTS>RLh+J&J96LsF$NBzsO4Ct$E3*~B-2(!k z5&BkY=%-@0*u|Q$hG|uk=3Mi}I%2Hscy5w8SzgLl4I2ZHzwDoCtCM2RpK>naA@C(m zpm!d%oqStMloSjdbBt%_6q`fkY`J}_S?S=J*@it7*P81ZwZ;9*l2{D=(0Ww!HfikA z-05`>5JRa*Je;%$G9x^0BdIma>Cj1N43ZC-_}QLGlwsKPG)8vZy109*#K)Y7h4Kd{F1;RJe{C%{(b5b z_pHSv+-{EGTwwB<6;}5ALmt@qk%y<<9`&+U*yV1;xwR;*p)Ok`@<70yK~MJMjbQag?fLygKz?j>j~lXS1N6M#)}lV03h#G*$nv!zcXA2K={ zi`#2ZtkMVH(`nD*YnBoN!yFoXqUJjIZy=pE$hjw>QCKz?t!yyJ9z5~SN|zS4DM@uY zZ?T~OuOQQHFJqkniL;KQxT(7;$ZVF%KIzA?t1cE!NalpYl@xrcx8$kle_F0$c}kvD zTk;3@g&e`kRr0FPgcFU!K9q9@DOf}*$%k)T{#0B!O5!0_1ZsLPrEsxM;iI>Qdz>j? zx%Vdms(7fIZx7EdIx>IgLYjJmS`)!EbMErncGH9GYl5}bWrva1i9X~t^qh_UF4 zT(!4|Ws28s)_k0YX6PuoA0s-|u76I|2`wUtSqI(#XSEs%&#|X>sWr9QM5-Boi1(~} zjYd0(44B%(j4|s}t<;xutQ?BNhgp_)j35ICkUNT29m?@Xi`dWmWBJs(m2Ly}4WHJu z=wz0sM%*dKrBXVAo!I3_;<7h0f3z-v&ztxVDTXnI1mycw=w)daFI}XD+Cz=Zgqo`N z4C@?h1|t|4??OQ>O&uIJ>Rt3Evb|f#O9d**aKf@JWs(4Pu~0}PdegUIb9Xn3R(yAjCDCJ_F9!=s z!OUBdO^cip>?)MEGCH0%jPb=Zgyea5iuFlvHT#hq=+UH`5BmUm{arYMbYucMN^2Fx^mB?mnag$PAUo+jv{{UrQ5}<&R%#yBS=cnx$tk49g?rHzPDy6D^$88>yMZ%e&>^E-9oY z%cnI~&R>xt%mB_Q4#NmE$GVMCm=bpO^`Hk8G{JQdjIqxHIG_g+ zj;66SJwQkZjzYMg483uKd_nN=1Rbj;fA5yAe^b0=@phj!{o4T`{X~<+Y=wh^t^^QD!DutfEXE+4ANB;1vr5U2rPh$1# z`hJ-RlH+P_J5&LZ$IuGqJZ2h5?JsTSM~X64g0^r@Fn>zYg(XqGW@T2Rg{)!fn&th? z+n+Vyi$5fBouCfgJJm^mxO4%4JdAtSqM;bAjz=j;?r7;+=B_S~M{v@Dd09Xp)Hd;X zNEIB#DCc(E=ChK7w6ZmiGS=s@*~c5f8_5VqWd)S>K9y3!>E*MANfp)E0bt_FZx)nrDd9L{QB4nrn$Uo*sm^Bz8ZCO3#eR3G-EmIUv-vsQMN}(#MH6lB&Zf zdI}?!H_E>Q9IaO5-o@=kX|ApOh2CV41yB!T{QB0ujBT#1&7}NJgE78&1OEW6Sxp+k zN1sn%kVvtv3oDZ&9*PG+SA0TI@1%(s2ZWX0y&1E~sb@_%H}5k&;%_i?@4UQi%$P5APn$fJ@t!~~V@|^7e za`??h40Flzt|W|b`wmzybBe7Qql}=gwI;o}n(J+{l}Jj4X*zBnO4d35+m&j^jMlEz(*@W&xFBf8zAc2>x}Hla|J@v>l{+ z1@kxB8gf5(Dfw9rL9Sm&)-F8nvncaml&orpx6P8n^{pdO#acwq45se}bYLz<4=0=x zO3N8(6=YcgoEApTf5Ns>Ph*Bqls+Y5IVTZ}Dar0dRb(jJae?WYqDi2W%yGLrfygc{hR5>NPFO~E~lz@Dp65dn}`-jivfiv@v3uT7s|02 z?OPcZqdr><=V9Q~0h`NK893sVfYE|^EhkxHBaVitL|Q~E&<=$5rD5L3Padq(#-yr| z-yLh4f@im5ynLsxH7+G{9VhmPm9{Y0I~>+ZNU{v6$fe2`Hhj{xn3A&hG^{(D5QJm7 zoP$|L@>Lu*KMGu|cRCf6FGnZX*DTkJPtTM1)8r0gqn7#@w-#h9I|0pe4R*iXKb;(c z<}P^3aVLeY3a~0(K1ch)x$hEQgW>tqZUnM?y^qW(sM(3jLzhW#d8%0A~O>yI6U>L z(Z>U@3kxa6es{$e9hqorTO`^Ld3htRHI7E>f}59UJwc~p8+SUNv}MaHEc5F04E)Rq*d7ExK39+d(<%5Ng+IK`p_$~%G@A49-fpgp4l6nX^T-M)MJSNmC4UR zSbEG#q0%x#6OzNJKjB$9JLqp1D{2~spl&YAGn3`I5P&ZgP6-9fF*sl(QNi{0H8<{P zl3N*e+6=8bFmOtof_jl#>nD;6f!md=GP_>=XO^EIT-$Vs`o@4 zW+vItj^5Pz2-^EW3RiK(Q!ULPDeBA`XtU^fKZQf8G2a$OI5iSAgC|=R+6wYNybfzV zDdddiH!JitgM_*>rA|Fjq_*uD%E`CtwF2rF1Ywvr)kRM%%I1M^2;6PB*(7K04w`x2L!XrXHLH!*<|;`&jpVk98A?kwT|Q#0 zxnEp*)muxME)`Is%O)@~IW>!mTbrq;v80;Su9k5aH##xYy!C*jcX1p#Fu-Bh{{V;O zR&Juc(=|zEK&=DgYqMhq&>r7I!K%Ak3dbciwAS;8t|2Pt_PwJ-|6dD8TTc`=B-0p%@nBRb{zg4!KAraZA8g7-KvFygT-65iHdGkIUD)+-pu1a zGnNEf$5S($*&HWB>l?vcjSn+K(JvI)_v@z-Y|Cqk&p#K*J89o-J?^>O3XHc zpQUVC$r3(xXYk^p;bO+0q0c-@%-cOqde;7*Kbarg7#@O)flYipwGM6@KE}5ZgJf(2 z@TYJ&iF8xHA804)YhL8KP4Y$${05c8Lt&vrNHRX}p*62!WMDTckHVe6hCM#gIfw4l z{xsW%WxN3T(-C=eYY^g3A$MmvtFiMC`cZM%jo1T>56-L1Tg>A$fSTq_u_@V~#-O_- zTLOC1LS@SdHzCSmpwmeS^H9c(IUo0D<;_OtB2k4Mc%q zF6P>}!RHwslw1Fc1arR~+T6Nuy>N;>1H%{C>eaB5F1M5YTU{{SMZ4GwVop`Q)duX;lu zx0J4>_Um>axY!f>ub-Rq;MCCRS8?h9M_9<=Twb4|Z5;tf4mf>m8ukAMFF zRb|6y+INU!Wy8ZGE2iv*7{*3()E?gS(3-kECMJA0nYTIPZhBA+S%DS8E9WCLu4n@f zRIpQQ&mYReo(EdinL|cF6am1QFxy;X9B?aY`$~B-(MCznU`+swCvXk{&M7V|jk|xP z-r3sZbdq7a@)@k%%?k2Oj)t1grwroe=ID>%w}WW>-h_v7BS~%*jA{s8q_7{(pt-uW z5p9mmBko@2Q0@N!e1U;X3Nu+=HNlWZyc~=$rrR`g+%m|KI%A9`F;pj^d6D5NB~7Kd z9D|yu>}_X|TeCwL#|VJrp8Y9U5Y0?w;?JR^x^pw3mPUL60c8XER&vnVo~314vW$z- zMMGw(4M>Z06*-ucD!(p9GAeG$vNG-;_l`XdDk)-M#{(lZvayS7Xf)Nx>rIeevyq~Z zN{0mqq_Az*0w%`U0a6V_w$!*gGot%?V{k9eIL}JSErHxt9?czI(V7x!Adk2Y*0Pz6 zQl^P@Bw3d2TmD%f{3?qzQi>tTp6zE0uv|Zw-fgV-Wj&FH`1ShM41OeKxAImEtBenu zILAupOt{JUh{xeem7((^PS{3;t(TM;DhbX9 zaZoIkZ8Tfw#A6=CXRKSO2$PWc#=(0FR+kL7@$|NQYlM@ zX64wDN$Z^TrpTz0mW{&(w|t6-$Qbt&$R!yEAwwTSO&vb&N99Oj zHu2PL{AuDM#B7Tmr1M3>#k&cNidB<4dyZ;bXp$5=Mp*SVjIGetH|k9)tH?GeAP#fu zRipjvnLQZi@TFng)3@^BhmaWOG~0jlhY1@X?U!w$m6%+PcoO4yo08ps*{+7uL5?CN zxju9C1NaBw-n5N2L#U2g*Fm~>jfr0=`Zn&~xURN#0z?Ca-HZ>Rt+{)db9NgHOK%E9 zqvcVKYIKFBT*fyM+r3tVYFo3Mh*}(mjk=*d{i=`>Lg17?LORlDX_nd`&4NfB&N_W) zxsi-8+{KjLkVbkMT8_x8G;Ir&Z@~J~B)HrZFbC;WpK@9l%)=XaKD7%-SKLz-i6xsn z924tMzsxcz0QF#bCkCO<7$@mK$wYx$`{Bc>>WxF`SpdCL-k`~8ODTuco z2cgBp|4${Kp_wFHziaSZ^&0WtWV}j%J*_GKX@dCe=JnDb6Cd1G<#!t;s_#OLd1OL zQgU!QXVaWkhlV4x(=8;sbLF((bUEi}?tZ`FPAjOS`XHDy>K9VR6XuZeMo%1nN@Bn6 z!*@K?veL2}Jd;G9G@z5wRdM}l2PNB|T0;_8)pW`CIYP*C`TM!=RrTovJ|DWz@g~9b zClt__?>ge;A!&|gh4#t-87H$3>+4nYOR=eGk;>f581kka@;V>KtuY-f=Z!SW7V=%B zdDIQak&;2hV9Tb>s%iw=7*&KJ-br%U!6O(QPZdd-w2@+e9o$TDWu`WLW^g}2nxSo_ zEc#}bc`WlWyqOiE1S*9&$m8iiv?IFlHnkz!E$zLaZh|!%zg+(SD$|t-J_{ z9k>br>GBe23!Lnx>h47s6W?63DR3RiDN)qm;A5Xk>8@c7s9N#&bmsnR{uEd)afnqy z{hr=9!zS_az350qTW#uG4jFg`tlKCuy)b09U%iE%NZ+nE9)!|7{!19+QhIf%lVv7Fs6OY9 z9RN5K$-Z!fkFQG2vsz?oY73bcZTpNnO5;npVg_lT4tYHd zPPb88xS25VNTo>w2R`Gm{&kaW?2S@gSpy?_8oX{5*K}=>OPTg7l^LxY{{VFURc+)x zwE38vQPc~-&dVAWDshp`W>~GzNggG_$aB4p6no~o?s?WajXK(AV6n{*cFug$+t6`} zscAe30C}$vgdaDQHxo&X8`7d_P@g_kMJtH{jDz@fu0vSyNRm}cP-KM%?##Zw-pBG4 zi)kCi4V#woN2hPUy1Gcjh|2_Q+1L-LJqgG3t^!+$CSxSd>`%X1Ih!llCDhi{^*HV| z2+^fc3^vM(w0yg}n!zi)VLN=yk-!3}E21vq?wMw3$l)@?_ahjml#E-cJia#^_MzI_ z70o;RMRDdzNDLHrY-bgzZDUx2Laao5*!+p5~ZbrEzD1UL1qnYfLeVASlV|$C@p0Qn8P2esZjKk{A5U6~Qm}g~j5BURGOg`Nq*NTcikHR7%HW?vRpVj* z0M|5fp}Dfw{mZpU1ED_Dh}r2)MIs#8GE3#CRg4@QhHR2Q!h{@g!8GhHV&%56eFUYV zm6U%F&T>6Ds=-SsA-WtAXtEAtq_y!fNiqxAQb>Wx%xVezKCUmrCnjNK!#E>eSxksPP>vB1d$!TFg~3-($~28yQ8d<;p5138;4NCk@c$+***Qs z&lAF{E+6k6arhq z#%YMkEuW=ov@t+N`(oT+CDyTdV1r`os z3r#IJ3aBNZEMDLAs$xn4#r2ckstFAOv1N!S zkPTLO)dpR_9;D)d*_W11E1$XXPMc|v*GSAihCiJCln<4i0e)+Tx$)RJ7M70w@;1@= z3Imkw-S`~Wnor_AYVRRrTXj97UVk&12g-U$xsDaxBZX1jjs#3xe_4(ME z0?6-g{9|WpA@Z-{Y;onpg~|GmYlSw@!}rRA@79x7E;HrW*}U=Ap*(LKDy18YvSa$z zL~u!R-d&}%vp4%IflHMX>dlqp)bA#ZmO`YTP6s~p z>}PWm&Y5l_nn@Z*jD`kAC#7{(8Xcv#jDE!`{h}0u%>DKbTZ&@2ExxsPG!eLhCPrN3 zIUnOzHD-?6Tb3xmRb)mg2OQ*d{3=>5F6cq3%9t%^T%YbZKhG5SNIZZ)3Pl*Y=B06Z z@j*032#3t8*KzeVds?;4nVm;5ki?p3Ov=&)#f|8e-a?^<9Z2Da(x$bOI5fcQG)uzw zcaRv7e6?b-F9fN<0RDAXLn@aVgoF6tMn7B;{V0mCs!24FC5QZr{v03l)-(SAji+02=c7*KjqIq%C1}}D**oPoPVVuqnjfebLssmYn3swkNRc} zA(fC}!So;2o?w(lWHGQ#7bIhzv;dYzGB@E>p_x*1+Z?2>1chD(F~>ns-zi6oGVMEaidID?$N|qoO~<7%kLS{w2_5JS z2XJWvdVpyUL3YxDzPO+)J{je4CwvSLyej)*vgXz9oRnGIvFr12E4Iwwx(ldmpv|qUN70?nLF_&1xMe_D znk|zb_D=mn|DL>^)wU8 z%QHA#qm9jy2em^ik+Q7JbG<_#^6*QQNP&Fyk~Sj36) zu+O12N>%c(M}CLB3EyH|4JPwq1NUqQ_Q)cx`H6p}n}%})&>$yh;#~GVpI@yCTG@lA zWpq=v+`#P^BfE4Tg)CAL9?&X`SsdY0w&Hu5d#lcN0r=vER9|sz9i)&B-2fG4)(;`p zEbeA;oKgnX$n=X?7F#7~r+8j5w5trOV09QZk89#f7rBwX#dB>cjJlY?R`pL}Mf z?Hbt@uTY3{@<^euT?iwmsH?DF>2l7oM}YD!^MS(;bKaV%ZiKj8!%0dK1M%oZbq}Q2 z+(($1p<+2Hj30b|g<7bY9?hK5orvsfM(04ei#}T<$&7-81Nl{RCN5TGGXu^AB$m@h z)fi*32B=mTPrvIxlpw<`Fc^Q{VffRFME&r6s1VJ~+0WxmbSiQ_W%U#Vhk1?lsQR$L zIQmk$9g9YGTQ-Q1mmr*WsydC>v9o!jg`Jd)D9%nzOzmrx3*N<)+4PhvIHp0xB|TTPq*G2&hxlEm>hmo!pQCJHSIdjB1@HKAHu&e{VR%(O>XG_ z0QbqIaM0*y@uXuS*4fw<;iK_J9d8=+ODM-}d zFPa& z1CZ@7?eS)$f4sT=RLecf;!DQb*eTZ^N@D0ePCRm3*4Y05N{{CM0EDEpk6*%pn6ax{%<#Y7jkcP%twhUqml2~Cl{vzb$fhG} zN01A9r_Vk}AC~H?r4|+!dxH$dK>1IhpouG%pe1blHlN*L{{W(zqkEAAdOE=ons`Ry zy@$w0;ns_blrCjewbyqr1?~U<_xVz@=88#{{%O^`#+YOdnc|+IV*UQ0qVDX*jN9!3 z42*G;&(oTzd3EI6Ux?E_Wx7JS9XUD0A=sR5njtmJf+YI}P$|YoKn-V!4Z)1Q5JwO| zJ18I4hQh?M+^Y#v8+B08DlX;A9CzoXRcW>{3U(=8SoMGKsTkctT!ybVb3qmy)ZRwqO+REdmu9}403BdJk6m) z@#%v|EE=LljS-AvAmmgRh|0;3dU})m>7XmpVEP(k9k|i8zq)@a0^;O%6nSr)Y;XJI z(jCRXJq;rxFWy!kQHm@VL$F04NkGb@;B`ArD6m})+8%c*ZGcP&-h}1<0F`U(d$XP5J4KlASzKI^<-qI@qMRD*V!oHjX)Wxe zlm#aY$~ec@kH(*}irtw!Ra@FI{7iT}KYZ)NQ78Mv6HPukxpDrFWAdJay(?1g4p^u5 zQK1ioZdpKiAq4jT=lNA_JH^_S)#P!TBZA#f6N))$vRtuidlj#2jg{oKp`)3iI3+uN zf6ADamvNaVx`Jgg$X%VrIVUyGC2ohIO}4dqj8z!UK_BkmikQZyY1&JAW~1b7uW~Vt zg@E@Ww5)Wg?xsZ$yhy|i?T^nDN)ndHiB_c#ic9+Wl_N}WNGIKEMeXFbx|Y?g=68vT zoL~d^)Hu%ev8t(0miHi7at3zu{>5oq=`u=J%=1x6=&UnTsZ@Oo9xo3a+&K{OD9<)xvnXaA`@4a_bzzGHsNR)TvND zzLd+vk+zAZno$ub`_&kaW#_$S(@w`kXBMx03lS_>1MN1Xj(9>vL1PNWvPyi&?eUh} z zsi<5WFm0^9gmGN=>UA~H_H(c4_c~-(aG4%A=j4fw)7%^m z4SBYYs@&MYaSKTxw2`+etCi>1ihk9L*rU>8@h+0;vCDHMxo#DhZT)&znMrXhs?Q8C z#-HDll6~@Ndr2|-D5KCWwX2;n{K59mF=RsU%Ge{0&Hf^|P}cKWMzTAzDBy-5dx7W= zr4B%$a^H-+LN#lc5mYDH)b3&YOxXUF%Un!kGwv8-z<_AGdxetN^gjV>Cg$FKI^g+& z6!Ri-a(4G0hxM*s!M9CgsLOMLQag4B>T!?dPBZ0tlPGg5qt66x~UCZ3~46hA&<8-$r_T%({h}BD%4qK-;8lU%vB|GDZ*K zOibkbPpgTvvX5*UJ?ry2H6 zW1IrVueCcZ#zQ=hD-FkUU<$vd+Ac|V3nn-RkaJ19Y&&-*(ez2C7Z);sqS*uC$idG+ z_~NfaWp_N%%W;VdNMTiBk@AkX2dD$R8odFrkjG{j%&@A<=7NioLG~x89Mp-X$8Bd7 z)T**8lBa7B#QL94!lun?B1uKP&Y>(NW}Y}cXomoAJwfhAOjWHi!U!GPQE6184W(Qu z1oM;lQFXEzJ=Kg;%q^j{@+WL58-8UyNzY8<6}b0WZI!!U-rI>>3@e5!B#t_8f1kA` zn({#u_sHpM6CCY~ZXkbOsi^jeyCbLS z(n|9DjuWlof?PF#ugS}ODoOB%5JapSQi|>wNQ>3C&GCfG>4N~S#;Hbgrc6t>0 z9i{RlvcQuxXL5zW&2l~olUHlBm(TwILM(ER{7KFa*P-cFrzY&sr$P}|jILR`vAnhy zKV`HL!EknfPUQ+e4hcWfyKBo>?``%zN~zA!*xl@ClwGtr(dPF0E!Loh#_1#bGknF4 z1FIA3+mT(2_HxbuEyT?H99B#f&m=xYA}iryRR zJ7tP=d7^gToyC`*Q`?}`90x#V{=h%?u3Ui1Mlj4$icS&dtlPdxg0u>cJ%K;16NPcW_c26WK`!RPD>F>XQ)p#(?(<-Pyk?e>E5@5 zVA;=6&c|HR#TJ<)HxdZql}u&J?fTa#uDH0C+7f)FX`}U{&KMhRFNA z=D9nxzmE1PFPSH|USp94W6y7)>A|RSZ{4uf1%7+|$3kxs!oW?sN@RHn!uA;B zKJ|4PO3j&3r!H<}i`rB}N3?nGZVpGx%v&IO0bJGeSF=L0UQ3TLfLQ+k`qhf76{Jn) zg_TWeJr3qAZ*8^4pxhYYE~}nE1CP|#EvZV8=@X(58d8z2KDj@jt)%S(o-`vWi?TPo zFF3c;V`)xYuJTKAPbRXWxQ^P`v*Tny567StQAtNqYSZ?WFKvwoCQB4E>JtMr?pT87 zl6wzojZL1X0>?O3QbQ}_J5TxG)VNt*8kV0k<*4k6YpLGrkl!P1irB&ertAPWuV2Et zLY8i^$0?2%jc^h$R29y84hYB<$tlZe9q^+BuLL(ljtDgI4Z1}kU-gA&JfEdj%nNYR z+t|a@0PG0o)GcJ%H&JED5rm}e9E@!<91qf?eTU{_Y!CaigiyHC(pgJL7>>j34cFSD zz0{$E?F$nyU&5~GUKg;{F0QqE=#0}`OkO?5G;v&&j(suKxVR7?XFjl*#JMWrt;h@m8{pyIqI!bOzDFL zP|86EkVkspd<(8i9n6|7**`8!IsO7Vr_;EvD;bwh4!M)Odz#X$r>4yBskyH-=oXTI z1Ja6MMI~64LI+i7zv8VwP@1kXgMk? z=xllSH<_HSwrw6!fNT{3cxnN!Fh5)CIV<4dudEvUI1aO1s9 zq}#H&8HDt2Ph(2&pzOMu8gj>`*~W-A&nVpNPhP-%KgO%-YRwv5D-yeQALxIjD|2dB z!=f}AG`f$@wqp#d^0Mcrx6-q%Zb+RR9wuBj)`aB|SxCJOv3+3sCd;C>;E%}75=^q9 zi9jK~T>k)dg=-m^I2}iZ{Ap=C!W-C97h{EW7zgX=TvJ*pla2_hRS0yhdsO!#IQJ;v z*A1e0m91?S%TwI(pu}?yNc0W8P+S9hDTcDZ*|M}B*MN-BcO~1Z^Y5g zh3wlcc-Apx;mz^&TZoAI?&7%3YgoCtK(ky-`&=I`PL&gylCh*|xav^2hG}jjc@t@K zxri9+&1lB(>Q?ICI=bg_uW{H`3B8RP>WZop43>yMBkz&8@%)8rSY5+mI=puAt`}TS%46 zF%kn~8;(!8J^8Ohw$Ng=m1T{T;FI@8Pt??M$+4U@osortxOa_)FhN|OKz+Eae-i8V z*3%t6#x!e`KfNRUTz_9y_7wSfb!9>hZpd^9_JOW08Y2N1WqcMhc&+*~+o ziR7Jtz}#@7=m!-%%Ev;(KeWqQzO4G4;)29M27IM{xb;52;Z>lx7l|w~f$~OOTOV|R zn$CB6v!)c(U%Fnjy2F=VgynyoW=7?vFv`d`^mC9@`te-+w$$wQw$ez@@eaDunC)!` zmm8<=`3MB+wh*BZ*Qv$*;u57O2r(225bS7k4|f@l~pIow>=I*uUWe zFpdjm<>O9R0x}5CLILPSbXK;}L3wj{4*QFOH!B;nekKoGb|wrdbv6sgF~}O)QMJUgo);_Aj>9$3 zNYLB>dm2?7bm(!xpaqQNo=!8^Q5eY1I+NajCl9x~aqURR?l>6yX@IiGg+@+z^r)IM z<+c=jyb=unSkp%(vyM$cXL7Kv(;oK8$2GMn#ygo*i~ficmvY1sU;_KNG6qr>7|QFwDs$0d675E(UgtD=tpr= zS@=cb0!c3#GEPFr*YiK&O=Ib;B<@SV&FYnjnPR(L)>14oAd8ogRMDRp*%Ufh>QZ5rv$BKO1kG@e@s z;gjzX<=~%wmCo_=`!Hs!2Nk@L$Xi?L`jO4N4I2Lde5qskn(poVRi;c;VV^l*qd6HS zvUuvx>@!7$rO}>vvE18g5=A@8(ndhTf(SYEHPUMD{juXhmgo#{z=S$q7gOs@-XITZGQeu98Dm)xxgPv=C>qNBEZPw<>7D=8^BtBsvDXqSuUMGv;MAZ2nOLim-zm=>KN{ARleraJdNVfi zVI($@1`Qyab1r=abb3#N?Ju+_)rXaMKy;fthUQ?4fMsTOf?sIx}pw^G&NKcoZGyEpB zJV&9ujmeN95Xy4DHgJ6nOe#3+Inz>Fn^wB0DycKD1MWz;C4a&7&tSgRXR2mVfom z{ehEk{{R}(nnV58{{X+F)@2;^-jgM?Y|=`s%g`@F_|~M*$0}`Q+&v3Z3$vUy@)X{n zu*cAoBBrr-Yn*TDY(&;?8I{KjlAj99^(XX~_A&1b(HJwH>s z39~x<(pS%&qUVaMsI>7}mx>o)M=S?HT{JM$tn|@$93#x^nWcBccacGicRsLIY#7u74>xCXvPaL^#3HPOZBftw zI{sCH={~=Dw~ur7q}{zi=okHw>sQUKj%610{EeexZ|D8(=I!XFIVXf?7~}ztJ!&~^ zmO0U_(rAzj%OfiE9fu#*rM8Q4M<3!Vf9wwu*;~A$N&sL6b>KJz!tPKxR`(2DqPtA-G?_Vif4C%?HdNG=k zx;j;S)oA3%u2&Mvd8or?rocK)C|{2?TWpE)gI8={YB#29>hc{Z&)4)koO-;`jb-8c zt9z$;2w>goqZcYzf@|NqJo_V`)~%M;O`1)aBOi&+O4Qa-XS2s#9p9B_Qtmc2r)49? zFKo))$|QURCwq4-=sz0P@kNC2Y7YV^`H)Dw`wZvYkZTWS(b4TwC-BCJX?(IvCJIJy zBFM@2HJhSpGU;YELUxn7RQ~?}wL)nFA3ZBCJuUMFRj1DnC9e-90X&J5iT+ zJww7ah#8*}@S(t7ROveMu1Z901xlLJtyK;J-Nj+;hOQEzNw=sPR_R0q>!bhBO^R~qz<)woUG5C z$r(_;?w23Jz={Gic~;=>!Tjo8<69bE>tdOGKyme^+qTeK+xk>38GC|Fz91Ok*Npnq za)29h^gIz%-N7pp#dcKra*VG|YM|x9lk_wu4ILhzJaNjgT*%@h_asw|zQ^0%vh3t9 z5sAmnoip53+=h*nI&Ym zhFl8iXwL8U-uyDg_Z<-#xLllp%; ztrYCjM}dHt3O}FcPHiP*T@>XmX&n`vvR~Q6B0J_of(gMsqv=(wuYa>G<-6QlB&=)G zE`QJITvCH__j;a!aZzq%__Zc~6xqh4n5@Z!bLpMO@jQMNL&I^qO!p2!DhlTv$^QWL z*HtU zHy`2ynwB<19QCPXSOUi%I|r`BXHlJGL z=0OlAoz21Lts_e$U=f3wk%%M=Vze^_BiJ7K6t2H0ARN#FAj=GpJJV72pMRwQSq!SY z1wV#q7bgdTPqi_YqY%0R;d%TitQi0$vb}`@5urfCXieGv5soTlSK|yx`cMMK1eZ`- zZeN{%W*(Hsww`OW`#C;nQsVpO=!CrJ}ZU@E^Yq6qB)X1(yDX^x8sVXHMFfO z-NYrjwMN|}b_}RD+Z;jU0h7*Z*l&Pp>R-RJPhaY&2L-Y zTM!XEvgkhQFCb_1sB%&JCqkg-Ys|OdNXPbiy!dyOi=Ev60GoFk9gzwD)h6Z&g8$6uu;Y z%5)P)?zpDd0Y@xf58r#{GeePFV@v&Gi92gH1RHV1ZpOaF%5A^T32l4b;X7V1xaISv zcU+HL8MUxeCX^*D?H!9(nfQ~F^{ET9p@oLERYrWp7Nj+v{!ZOq-;L9g3ipkjnVNbw zQ@2jX6Ieg_6Hj?$cNuTfSnp3FC!sAA9vWn^KjOquOws+sKf2;fa-v1I)4W+-`5maJ zn{}v$H3kc^b477u$P5?8hg}TF{rQBKCgWkM;C1=Mpn_0WdryR za3MrG*?56Qy_O$}?IvDev#r)eEF6?ZlaKfFqEvhacStr_WaGeRFSlGpl6GA{b3tBS z*GPT!Fvml#eqW_$5&DuWXyRigOYjAI;oRg=WFpO2M9UziENM%}8|Eq{^kv(yqIry& z5Ql@}*vB(+y=bo>%Clh6%O`v+9&0$T>hz&5wk&0M%aR-KAHdq`SxY5ugqyd^cr!|e zQbB;md?)|T=vAL@kA~z1;!1Alau+3St7*>gNHqS68siO41~{keFUc?`SZlSUz=zWC z32IDnjLMCIsj5g*pyXkFG*w0V#!E7{$r*!QYXMvmQIHYG`KqlMDz4!)t^@0vNbMBT z?SXqlo6rajJ9wD(aVGX}={Y>-%bEWGhdT`P?{j9t`3sqfExV*^?`l%;&BEk-3@Ee zt+Bkq9atujWAMUoLi=yb%4lS@WUltpzxVcI;tqOy2CsV$uD8Z;dk3VqgWX5PdURDY zE>Z72C=Z<=)H$Ug<-48v&w5&I%dV|puunu_YD@BtNsqSKnylaW6qy9UE;x~q9&|c^ z%GV-bpG24up&-EPyfB`tOy{O5E_AQeCF0QkOO;E=6~**Y3x+(Oe7Z9}@R_xKUj8;Q z1wkazSRkLOx~`qSSaV7S*y~KEt;?5O9UsxC5S5aU;Kh5lblqGDHYBSlW^nNu-O$!( z$Ne>%SRfKfZ1=ikI&5PM5=qZ}Nk|MiGwmxr*#F=?G3Z=)d)i3Dp`NdW?hbgU_u6Sx zxO5Te!9!tMsOc8>@d!SFnG(_HIOm6!B)5=Cfb`Z7ymS9uO2!{}G_g7w$?-&lNZ9z1 zUz3;HFjp40uq;=yjEc!ZoN7(k?iK3PZt?Ishb!*^tD5VYCWX=s>Xb3;l%mIdfhrNo z?sq;)2-VaLxUw|?l7`{nU1F?KA{1oq7kdw_RoO$CL$DJ&P@vi)XiH`+>^q^t5<y{rt?aMcK-uvVtLg zG4_-cNo&a6`c)rX>X~n&2C2E#t9c(u(XgZK3;>d6m%dkw)*R9r>cS3M0;c{GYlKmfZh)AN* z^aQ5G9)(FBF1L#N*M@KERUm>`Tzk_-V^ckB9qrFj{F@v8^bnAb`eGEE|Ff2udL;cx z@c4&ghPJ9u>#?BNU>l$1_-{(BSr0YMU`GRqL|xUQn<4vh&zy8>Cd`g#TntYHl!-7k z!LrTaRBs%YXQ=&UkQzu9AJb(M_`q+)+B+{E0?;8)?1hcgK@GvMNRo?0o+r$ql$ItBYGJy=?QkK7@FFQ9P$$4_%)+)~ z`77mf&{s}rU*)0oa$eXI&O5lXXcT?9vP8~*5Px?BJ9%|$Cz!weToGAt)mj-4kQH*V z+uV|WiBoawlnL$fCxC0IyP)dQwa>+6@z7cSq!#2pTC1fyy9-vW6x}NCHqjLqKrrbU zf^l=%Nj|(ZXv@M}Q|QfCpxmb}daMU5VktKk%~+&~XDZ`OpUGs#wC3|& zj&c(hYW5~~yjAay1EX;*bRh#0{r)a2Y)@*)FcBOZx^3ka?(SY`?Z3Bhk2#g{Kj3rc zPxBRVsjf0(DZ5>nCo`N8@^U2J#Z#(j{L@BU3=mS=#YN1?Z>k<0x>qm!Vzm;}yP1u9 z-{x{E{Ba4ciyJ;UolPMdBJ9UOdIgMlE8?x9Rn>II18Dm(dcLSi`HJ;@R8KbMjdd}( zK?bAh9ph4tll;tQ#AVm%> z;*Q(qfHN5G}UpvKamb%==JyW@njCPNd3tLvj5c>!j;;%IIwqpI<|?VRqf zmL@)7tK8$M_lv%>U&Er`9xBIQb6yRVti4isG>(a9DyRM5z`1a5ZOT^=aq3Pp_t${X zQE%TF47}mZ4W1m`1Hko#&-rbQU!3cHal_DUG{Be31-rRC5Zc-msecbGyJ)P5FL^%~=6g zF3S~e!$wU9L_RV%qef&)4;?4(-0RnVc-8oOQA|*c{w3#tu29nQ^lOk9rHAT<0=Axv zE6DzlcGL99_34~FUC3B+M)TWDp42lcKgnA2PUA}E@-fSmJHIE(+_U7&C@T;E124`% z)Ut>kOo=tCv})?^rqZY1D_o2&x!~x_%^lUMT)$ng#r0fm1mwPuWGiYuh)jC)tQ4<| z@I7;=L3gb3lA5V)JY@|_ixP3?K3*jub2kTGH{p_kW0Qxrv?uM_Qnl}CH2k8<$b{Ro z)pM7RR-?RN;cS0y^z7ddg7tEvel%9rSXd{vf74r&%lb^iKVPBvijTuJzX8I&Bz+E_@^eS;slnO}Aa0WV+1J(0<8^GM8M} z%vWzPGv4}Ii@|Yk*wt#&PAac3PGIx!v*He>`Il0}cJ|+oftJ>;E?0M-IxkmpziH4n zI-=H&(&(h}wZl`S0(+M7^0=M(=6#vvaTqVq;2)}0xmI&LS;xt3fm;f69gJN%wAunJ z_*F9%GDfZPe~!C=o)Ga@DqybKp3{B%f5mG7+sLMNUybrP_JanJ_FI#dW_DE4ubcq@k- z%Y4C-WeXDq5in{o%*c)$9P2kGJF=@N92baV>Oea^7Z&VLJ^7cA1_)g=R*NDjFsCMR z4B|OzD@a<7sSxIezbSb_g-z4IN@*$$U?AUD2J*EcUFgy5IG@WzyhkN#^h0IFJU&zn zV99Hf+=gh#MlN<+gnE8imlocoKUCjm68)qn=B1JyliM+mbV}W?@v^C!0y7C4w{~n1%H7<<5Z+#b^PSpUFMq}bOwhhqVu zL_jac#dVat=Y_Pjhc|Lw?xlHI+>?Kg)BQ%=-%#zq*brr z7-P&otwQO}Fa7#PO=699Ul%qhl22BssEj?|N+a2GlN*`^Vfj#pl_EF-f>Y2KQK(`0 zq4RslWFP@CPN*0YZ}2udPk&M_>1pcBu2LWQ*4qhm4?mY-TE|B3Qk3YZ)l(?9d^uek zt3p)^*B4AXfUoGVBXR;^6TFTVq@g`dR~EP&JfaZnE@H&@)%dQ#ih6bd9|*rc1>4Er z&0H`iU`>O$RrbZM!k(fPuS#-vHtPYNu!{9pny+`MtRa}UnT=F-&x`YOO?U^r;Lf}p zQ}NH!OVv?Qg$W;ng%Tx8rIe|pGgZb2?|YsYb|tE@+r-LMS{;Se;Mq_i0La_naI&dN zSPEK}SzbBHolez$QPz0uF+S6zUy7-qvJW>T(B5FFjU>wS9&Jy?0>u#xhFr5EwVb-* zBG?|o@F!oxyrUx}69oa_OE}Vs z@~oA%U4TbACjRxyPN9U3n~rvbZy;F)LIW{!mh)D=j^w1k*xN}59+A2*RI*`t%2YI6 zE~Z7f=+E^e$L$0jjN1|@+*`<^P-u!q*`Drq00zq%WHakYH`=?(dIL_o-V z;nefP;*h_N`J5|Xx>BVpw?+DTTVMCc{+8g&vnwZLAjPs5ULj)ZY)ii~mUitjaUx%@ zk*L88M>XKL^in6C+bok5RRetFTDRve7%_vKG3B_R#^Fvqu);%)anRZ8BlH?2&m=%! z9m?Gea6$49km%U{wU$G1MrbD>@}1$}*Ml5QQ-aF<+hpKB0LF&#uQ}o|AF`<_uGfKI z0?Mhz)-ZEf&0cP`TjJSo&{-z&-$Hw8OQsyGykT96!2DAb2Ybyw#=M=XGV~{-J_~-9 z2P}KLrPP)WeNjO^t*<+MBsW}ZNO zi<4^avxenx&@_j_Jq%AKFP-wiUZi2_QSMrx4mx_4?M6?+HAg76!Dy3I(eL4T437}w z2l9g@o{}0n;;|fOU)lu8nhr0)SndUhaMTMAY1iHEcQ){RCS|9 z`!#VXO~b{w!*=;1pWC3fsb5Xmm47*zMt!H3E};RUSGLoDRs zOY#TWx5h1XN~ypca5Hkw_(tleSah8dKCsUh1s)y;*Q1zPK!n`TL9;A+Ck{>kwN65W zOri{)R0bfD_|4;b^ZmrNdxQreWKaE00S!8unh60#6odj4&=k;>T#prncWfkRm-RC= zO97Ig0dq?Boh11zw|hnnBld%wo76Hw(S*p|0_Ut{65hwr5TSc(qzgC9##=g)akdJ# za4#chKK3A#a_HDw9j_D5D{aq=??Z1@9Wxl?$A6HXFp)ytLz(no2Ce-1$Efe>U2nBs zg&~a_xPPlkSUnaB&hCI$geenNXLPs#rqR7%Py`@mQVn0+yN8o4D zPn>wD_y2#-cSh1hE)?Sq^U#MnHIwxJ?;%CuAr}hg;M^UBwBDiHaL!BSolL+r;AtWiX{f>$`ci^|{)8_K6&=vQ8fTFk^ zt1SI~%+V8clN!0(dSSwdmj$)lWg&*;M+Y$3W>=5iB|rJy4gdR_>rEZ`h3lUW@@thJ z?FYVMtljqYI5Y`mvfM4H=f(x;BB|WM&4s!&E9FuBfdn3R#(fY;=i90e+D&%tuD1LZ z&2(UKCmd*$!;g3IRBRVzs&+p#XQ_YBAXPpM))QE?*EtHczPRwH))~46eikSH3$Opd z_*L$1eH4-WmqeXBv@J=l>C_W^E?df*itS2KCQV{#I{c|lnZ&_;^`o;lt!?tNoI_VW zYqgOpgr2)KmIQO1H@lr(<~z%@!gj_5yY*MY?r)Mxb`hdZdD6r}NP@C=!tco=A5aan zx~T^86iLIKIZY&C@-IaD;Hby6-bwoS`fNk5ZNpw5h1Fv2# z4912tPVNZNmqY|%zmMdL_|NqNzN?9|Cg}0p+bLUbRc$YoHg(4_#~vqEB$!L5MWC;A z;*RDeMm{+u2Po>Y>17>l?&?`!0o6S!LG-oe+|zGa0-{68y?aH^8n%9mXv|{THuRp@ ze;03`Gma1B`?|ZSn^`|o`nX?l>o_AGtys>pes&%`r<`I&&l_#R&MjHbc&md@=m!lH zj?UTYFVyqqw`&gj3qJCAa0W^aU*x|&6tnGzAgnp83QlCB(Y7V{z!1DS0(3^kx1*T# zHGOqi4dffg>wW5J6#CcvM;t+iZJtsJS2FQj&C2VsEOJFBOl`C1tk3G1ZTJjLog-Q3 zT2la&B|4J_nbogtDY2;ywZj3)#nmThoaxDmk{S8m6nI?LQ~A;aNt-W~nsQK$%`@1< zeJX+PXPL89sx7~U!4t^Ri8xRs*5ZP2jk3WiN-_Fd?aWjm&p&NQ-$em6Y{6>BO4$wi zvmN2`!k#H1PqlUjFNM?912pENgB(NBo@Wl>7U)(wji2#C_Q|huYo|C^Jw_s(FdtF)evR%RC{hk5?XkN>huK>f@?R;mM^Q#kIk5NlxV9#& zDaRrKi<@GlhiNAg{*H?NsSVyQUZ7=yJxK(Lmt(3lAL7JGdIC{*UAFzT7(r50Hcw@En(1$Uh#Vm834iOjN`2J~%Zb0QQbfu<%(E8u*C= z8|OkDhHv3U=4V1&2m{pjO3N_fIG_JH2v4Mr^F(8ec~WQk%$P=D7>dND`iTS3B74fj zJUo6jKmi)13eW#hEG$z$@#SvAC^n+UB{@9b>QwJdr7XDXH(RmF6JJqNy|}=Yz}tCR zE29|@9vogP!*0iGw-MY1Vl9!u-5Yc7e0)ABO;#CLZssvjaK(^fu3Urv6v*HBbk93Q z^gwm_T=3xLRSr&`3e^=xZH*6|{AB0tM10#AON>*8HndnKU$b2$hscNPfCRdy3g z?L6+3bv<4C*u9jUa?3gBhhArY+)u@?{x769tX^dHYg(UCcq+me z&O&d|y8Z#`*xwR~3g05@`p)D30jNKd3)OFuE9$(Ko~rAYWT?Z&36yKd!{D?;JN-s6 ztQA|dToTJ{@!`rk&_vJ8lDN}E#xp0U6~eq#r;HO1VmuQMVvw>K`}QKL!0`_tSJqER z5wv*S#|vXqM&4an@^O_8YqNo#znD#ZT*U$$4a#_>yj>cM#Znxiveig~$K{*a&`I%J zz8P+OzKf<)m4eCzUj0(?P2I7qLKnH8ah$Poo;)qTa}oFH)^;oHmAo2POcIs3{NA~%y~mr5Gc=;%v;<|yH! zvw9=w)R3AU%*jr{$!RX|;q7u4B?yS?$Wb9kc>SR@(s-#L#O%>pA}7qGK&X|fwmr*# ztS!~35A-L`RK9^bbgW~>i2NfzD#MAaa-a@7r+VF{m%2grfZ@mjmw}J@pw-w75+N_w zj2Yfg80m#z#GGMPe%M#BV4=9tBtmoVEIK#%8(GIM+|_wXopBtOglmCu@oFo_@%Cvl z86+rbw|H*>M9F?&YJ5$}8ZHFz%jE%n5<-N$QO79t9nJ!FY;;LJ;KK2W8IEx~3Ysu3 z1Qoyy!4X)$%Dr%uApwY4VO*_%A#hh5hyy&OF#gm@(_CiRf;{xjwuC)7wT#yX+@eo( z>AE(OE?Am!>1z`Rp>(=g(}!q?vWzqLbx+3Et#xhE-IM%G95foKf8&la;eOx+EZ*E7 zzR6gj@)o!ZZ#=VQccu$wSXb{?z=;b@`0=Wt8~fwC^`zJxX1s3%LD6p^6~xZ<(mwfW|6ZDM@U+^(GAHBSN8KK0VIGj#F`_<83L5GeLaz#1dm-(WoMp| zpvYwQFQqsCCdo*pVqlaUFM!_eo^??t>wV@`!fa6VI}g&gbB~XXeC;X`1mq+IS!|*- z&x1URDc#@agJs0l#?RGT9jLra^1srj$0a$58E}{PD==m3{R7l%mCpSb63xSY&FcX8 z73f_xhz0};t%$k?ts77z0xip&CDNGr5$U~rm|ChO7f#nf%csl3{{Tvhdx^VkuTm}@ zU5N(q`Nzcq>S}j2o6y`WWQJD?Pz<}@#0JP(RJDG5)f%vP)@}=yzocEfA0xjAv0||7 zj(8X$1ux&LaPstW32=xE+>Hk7KR~kmbGz>)s&XvN1oP=%@wq;Fat7_U*q3(PvUPp= zIbUTJ$jbx7IJK(;F6k1OtWFnAA84OL6fg`okh`sOG^NqF+hmvq&QR6jPV|(trqWfm zm_{widB*-jN7Vm?jxPTnI#xo^QTsI#f{r_aeaC;UH3b!&5!^%gReoEESN_^DiDDa@gOGci<&1`ped>i&Esq-i%?&b>0R}Qi_0((`hvu2|v*6pAf&Ez!TCvwe z^cc)E#iuhr+|1qA0@D!{bSsx8qtVHs#x%9ikfkYIemiGGkHP!z{FEkbXPu`Vcg9hD z@jW0s;S=i^AYwnvL(MZ`yNH=quNKGglhrzJI?5OiwEjv!YW`8?K-MAn(Ke} z?P16--n65|3Pgs+ZUuC z+U?eXa;&Mf?sbN+Bh#yA{cen?wNV*X0o+((_uyR7x!3o()QT)vW0;e!$u3J|%fo>I^Mtdf zU5grE84obo?s!Y!erVMhdavhhH^17Vwm=0hb@AeEvUwIx%y4KNR;Y%4zloL?b}W9^ zJ@s|@ghiV&Y_P{ka6nvy;I-Q}`=VAuodLr@%IN-j+-ID()i>=~Xlvh3AuZjWb2Ibd zh;bq%V&~+=6)^JUzt2?dp;|5=9-2}dEax|zBN9j!;}PJrmcK72~vQlfgt5+(#Zv^+G`%E~oN9tuX@oA}`^=y>Ga|FJMx>wG9 z_dQTKg$2RY^5Yis**rJ;yi^B;q*WzE7DkOVVapm%4+HJ&!pOB^c^JZU*ol({euc#$AoTR_4lFqZy|7F86>7tEc)n z`<1)o*5hkzW~aYV*TwzYjP62x=^|^EHnqSpsaWZvUd~Afr8C^A*3_#?Da>J4LYsJ3 zFeJ1(tBcWf+7p?SH3hb`1Q(tJ^!jgg-ay-GkpUwRRxeTZ*|vJTaO>NWRa9vsz>hV( zs@eKCz|_(!1$u1dUaR;tvC5))!M5(WZM6;??a|mcXgH|1b8=U@i&O~LhV8;v!2<|W z>wcn==GU?E5fmmgfESC=i30g}ry06q6@8SQ@t~Z*eSgiFYMITC=$3}>0WE(>-}-ho zRMga$u3W4mHM)BoR{k(G{!ZFpCjhy$X2cKrvJ!4%nB&uvvs*|N$Pm{`0`cNqu$n?w z_H5SdA#6V5GF22Q?VIl5xu@RXJmdnQdCRZz@|rt4k$*$P%N4B&nI{fb=wkpHI{1W`^DGsg`Utc2TQ!FmT3)i=N4zZ(v@(+7VuOule#5LBK}_aF z8&5P~KB)y_3UFxwUfV>pZi~WBh(6 zUuMuK&Ptmsr5$Ut+P92E^$(!W6vJx^e^KdLE~M&=KHtnE0{INm8w6TR)6Z=M4SZzK zP#+$uwW_BRm!D-jl}}iG;XZ87$=_iDcD{uc7+lMAsnl-BF%YA6!vG6ptTZRK?9WQp zpR`yATd{fUui<_U=II@Z+$3NRi#pXc&9Pc$qw?ehT*kUT?=^a=S$kFScH7jValCjw z&yAr${bbT>FK2yURF@JXlfkS52-f@omFGe3$v8_X(VPm)O3pZWxLU%mQrj*&Woo&L zs?*YfuTt3fw9boBX5jfYBYg2l-ozIu07D-|o>TiyxzpK(sr*}Gftoih-H51T)6xI( zF|&2^$j5ecEKw2#DhbYM!NtpoVc!9d;4k56T)h)5D7!gwj!eU~a?@B@rpLoSm|uL7 z`|nC9g&Aj0rj0u1Iv|eJ&KXk)t-f=Z9OaA2?YR6sZf(^D0kwTT7|;W;nu!97>2%t~ z*lC^N=9!s+n#PLODZr)KdaE?uA74?#3}q`6@Z3C=+Ce8wv1Av@6>~8S61hHM=Z%3xSZACFh#(pIb z7%b%PbC*u(p%)kh(*)3@qTZ06ZXi)Pk1(v8ZS2fLTW1OSQ9A`_8N=^s4}|*5qEgT@ z=#`RaGc(Ke3$HAC<3KTPo`3ds&!m7DpV>hq06)3Jvlf5CjWSdxPI&X2P!QCyh->AmQ+X zO}%By7W?AX#e2Gnl=B;%>;QRvxs3SN)>00AHmlnV0O0{eB-An}lI4oP=7=j92aIn(2bg$;YVk50CqaL(|> zah;0dE(S&^BqeWc9%2jPX4LA_n3I3voW$p>+aMu>bnVB{39KKE^2MYb1&-M{3Lt}*3)rCsd<85%zO)T3WjAczdlY(I%T=qs?k z?O4%VivmF1io;hp$71%S7OWP%vkG=J{gh(1yPaYhk;#%CoPV_G(gWli9PUszQpBD? zda?`Z1zATw^2)bX^*kHIy&ia3D|!!H zkyIJSv7SXWAG0gztC|YpmZzRRq+EwxOKu8D3X+d#(^jSe+Tve?qLtQ>P5|S+81!zDL2nVFz z5F9gmCgK%;hBpaa!T*pA6Z28)=u^bLVxKS0lJ9lUs|6@B;L@Tc=jfd)p>wk@hw+U) z!_TMwxHL#bfB-WCdB}Gn%;0hA*NpG8QvU$tFy{*BZ$ap_6m}9-t~UC9HW` z_aK(Z=*i8pb@vXt$gTPlY5#ekglU)0krDeutJB<8h0yRvsd>*sBPMF<4}jif-938tRAthD9>W% z9DGb2U8D-DNNgE*?&L!qVlxdz6;{ln-;-a#o_Ndg?nTr4J~G9Cbai@+|#N4F2qlE zH2bYKY$gl*@p<#|e_Dw!QXb#ndRB4i^iD7scE$HvXfg97t&s{mia~jXq~De97|B5&-fZH(i{N}JQ%4j5F96NB#Yskc=-!9-w!AMZ_#QPH?4H}J0fml7 zJqDE~#%5CPj|MK6rD|CM&q$kNWN*5>jso_X)01puxP^Vpgt>L?p?io+Jg0PR4FH!9 zINK-cR#ddE<VXw2&5lXDkx)<61fQRKpSqVYpy52|clqFM}>d zIclXCOl;mVM%Ho4*Tv6eurCa?l&~8A0N`e`zu4*Wb+%m}aOSB6jW{eHIpBI<#x`~+ zOtL7Q&m;R=Np0k!OIQB@(3}?jEdD_1pL=ijAw*uR_9W45g~J?Ge_Ge zGPJz~eTbgN*kcIXGs|^N$bJJxh9w7XSH2`KR?A6F@xLK8#DUNpSamfXX{y@Z&ezEi z!k5BRT~;47+pYBLto!PJc2-DYa5hs)CU2Qeax^AC%GK&F{g}N( z8okg^!>sz1Bn&k~v%sbZA%vgc|6!b~JyH3GqZEdv4(&Af{adT!X@CPCP+8I73g1Dq zhtEB!G^rOX=2KSVD!``1$35_X(W@_;mkJ|M~jQw``>-QynrF&SbTKJ`2WP zRiVyAA@{?i@>&>06=pTeuLzS$xTJ8w;suU+Unq)dlF?cnbhCLP;z}_YieP!7!-y7% z0Hy!S5qJFmbJu{Nt3vmXjMyjqQ}qd2(-w9^WkeZ@G7Z+gm;0CbwL7;K3fqDEiq5^7 z(O9(^$>0}rd$}h@>?=(x$^QR~!0<0O4`nFpi!*OZ>e>ONp+&Mxc!&3#>3=lo4Vwu4 zLzm_mpP3BgXF6Vw(n5bY;KbIv%o}<`F!(^6mlu6XTcTv+v%DxMVf*C1D??mLywA0k zUsC)OHF7JDW~~qKEBG5PS~2~V@pW}+)>TL*mX-X`eWJX+6V~TlWvw>2tGM?*vBFGq zv$j*x{*kGEQZPO)brUWLy7XdeeZvUZ>US)88phLynkizfkM()ZBPee?*th^|-xs2x zp`kOM`FvV1lbK)6m9;vxK!Syb;K-p`S70j|$N|%8(@kwxvT=5iwKzXW(ZUr-uW?@; zv8|D2++zb5N;F5ic$mJTHoEBqVym6kxq}uMrH`v_o`DS{!Qz~DWUULIys$c(+(fob z*YfhrU4z3VowVd~7|Gge6^yx=Qj7X((-|@_A0HJ%tHuft_wO7jsu0Ha7(B?e&(h1T z!f6iPy$4Hkf+vvhq7gEJcf^T^DWXY-D6_p9b3B0ocS_|^w(cFwpY}!1tPxIX1bUO( z_E7oy8-4Mv;N<3;1993#>Bw^-VY^S(V#uyqtEHp4Uj?sFpf!x<-+VjO04KLVGTS*G zyP%+?bKTO~$*=91##Y7myzHweEm&_ z-Va66bR5Ek(-|G2f)$kA@SsxIha$L!pXDwYKc8+s*Cq^=7k#QT8w{b!UDI8-p9}_* z_k6#lgnwD~1QWPS=L#6Mw#F@Vf@{tn186AgkVWc0>Ij~F-$X^v!=h45(&onk10+~3 zlyG6vg2U<@ERFtRo=CAJ=$t9tvzs|~Eq5WLnO&SiNKIJw=>yR{SDul6`aht$NTwVK zAPxuEy*bfqmTs<|Cy^zH5qy*-R~b4G7cWrVv2lJh|G~b8WK*=*mbayvJzaE_B7B68 zQ{*p1MNbSbtCUns<0pqZxE!_ez|ScdJ#glTBd(s8hK6LBL}16Y42x2&@1-tZUsB(| zb^`qd&Dm+5AqOF@Zwc#0qi<8G(h$MA)k}y5Rv$#<$6>3Y)*se=okYH|dVk7gBh)LT za;=*4NUxz%Z?#)tdkpT|4v782va2_zHcO*J8gysXNb~Z}JR9W*8Y00ST1=l?-EA%W zw)Ms7J=;wW29;zo#OgdnTv^XU9)>Qj5>MHk*J}BrGDBr&(;s_W3D@RA(<1yi)x;-r ze@g7*wDwA))+~@oeuQD41_t<>KxElC<4ia0f1TGP5wpLvKmDR(9e+S6fER>dy`-IsgC|Ryf`5 zj6J|{76PBP#TYT>vnOk7ikXqbxm|g#MHDo5B%8^pQLgoSEFgb5wzPHf6g3AMG8gle zA`EI+)yb(p&r2Ljt{L93}%8S#?^!} ziH{bKk43GsZmnILBALq|%HO+Db40nV!B{df)R#J4cjA-gq#IyNEVCrRs!5Pu*Yj_| zP#qgP3EWD#uJUOatWLHw*`ZYaZ8%kzd(_g6*{OC|sG0f( zFHiKqr@=V3B4t8gLGh6sfWTG58*!F+38&G#m z!Amddh29~VpVHV3u_4YZI!-1n9?L(#{;qw~KR|N0KGN zCq|sNmhZD|kl*CUa-K_Q@ZVXKrN!-cEKYSt_V4Q2RB_15x!;b!wAmSN8F7%lVoe6s z5KoowZ%3)}xr=wots)E3O)q2)G5%m_{G2{_WvlNFD9LIC#rc zdZ;!c0(G4)$2XJ_$tlGd9&jUV8=?3KFE(=~d@2~Dt5r|uB3|YDM6yrUt zVp@NFz&cvoUS^;HJo#ws^!I&D)6*wBP25~A_^JbY8=>{=dol{96vPgHmFDIV?3BGM@&(1Y4E z*4H;U)hv!!akt57m>j(QJ;V-PcT3{KN>@_TikjznTKe-g@q-!`K(OzBAy(17$-jG^ z_4Q1Ti7FX>G*X=N$!g`c`q49h)*5?I-EdsbMigE?6tgGIgeFD5pQ^e7fMq3UHi_~< z)r(}3)iy)7oyN=BDpzS?IDk%Y_8@R>bu3DGRg4KuRUL*a3m}d1G`mi7e4=^(Cgog^ zkeL3Aw05#{Xs5senV&uMmL@ew`s864=VUmQzUC+-;YVKm^?+R6GbQ*`&<9OR^?c|B z<9o9>U#|3YJF=7YFAp^xc*U2*ITjW7vcZ|Hp*&e zT|&6LKYdx&Cc|0$QtDsQ(iI+eN#TMbaXI1wA5e8tYiHF&;ETw}6!%eSa1CivU4aOb z*#xgE&^w%jW}g~=d^^?`o$8i|mGln}z#DN57}I%Vn*p6#I%p0`vga^<@irBjzpVfA z>A-Uc_q}S9&_Oa=V=3eZ@L{s-gZRfxA(jl|^hFgc`F#EjW?FMB4~LGuQZCn<_)dXq zWRGg@4be%D^Uh9zQ(k;18h|F9dRw%%~Dw;W6HTvNi2mCrF@T1G# z7f6*R|0Wtt+v&<7Ezfxy45@1G+BiF@`osk@ogD_$%+ZALg=DszD{44p?0igD&R2i@ zu;b_|6Ei1-)l-85c3o`S2GgD4(I2vXg*qK0(ZfoX!&5PC&mtLjdU@huT#-PPWn#(&>^}z1c9(>lgA$45dB4PGyuxfw1g}*Lvyd z=5Ekh=_FDUzD^?4C2$@1W|+LgW+r+tX4QHPmT$IaZ5F6%L#luQn7LWosg|j_LDdaR z|Gfgn7qdTT+e06Q{sHc>bsre|_Vj%J%WnJwNbo@22|?_DCOxFk88N0+i})GKrwl~s zIP&hhE0g>(pn`Z|>Z$AlrqTcL&PRQxPW_pLm4`elorv^5>|b!6`C|`#_!<sZO=k*!P&E*giDF;L`8>s?b(brHwN4Jf^=D8qqyf$WlqUPhfUnbeHRiqQ z*~i%**Yl6&A?{+Us%x<(uLFOzH&v~~!8)gn}7aV{sCC6Uv!<@JDU=8M`zpGnZvRgi%3Bfd>0aQx>b#OeCkuAod78veZyxf z%<(?sC03NRZf^E=I{l)V#wu@i2I6&un<9v6W(Xk`x zmqD3~EAvn!=ZV;9rqW`r#i6FT(k@#zG*O8mZ^Pged4*wBL0;8TN-YyrY|WOWXNA;wyixH9v-3csgd$15pmioN{qQa3q`&sfw@D`JZa z-82nlaA0cDd>3*Lm@qXRUfs=tQcT9(ymFa%QRJUbVzzlAD5Zy1oguAc*;psHhNGRX zEsT0FubW^*;a*hDwu80YhK7_nU&$fv_d{@Y3fJjEOP~L(yJkD_>dpeF*w0e*+={y7 zdiU@YNGjgr7@LV*HpS1IYMlf39hHNN7fi54wc{z<>3%rUR>w~#N6v(6$ux@0Y!0Ed zwy_Kc^YbNQYzS#}A>-tgqt55h&xLrz?QQ%``#%A32#)tIrCaD!{{ULn0bmJp=6#e|%S;{{U>=x86km0KQEe z$b78yA37t^$LCxE-`qd^v;2cm%p9kni4gQ*UViuTzyX<>EcD4?nq0xkcRGtG({4^j z`mhUp_(z#tGt%V6m?m1x03A>PYB5VMu1m&oX_(A53qxJhNBp%z^+8pF=AFR2n%(0tE+Hg$^7*Rq zr2t?1NV@+3kBe(*GyUj+rb3j$?4H`r#yLc>>DFl=~%*E z6}d*EA{^@`|nW=Q$QV2i?Ac~iAl8ZC0 zEHA9smJMCZI)$y|QLEaC197`JB9}1v4h{?1BVZo@dy4CB{5b`}nIm9_Jq<2o! zPQvO*S1MvAsZXO^Sh*^$J=D=;%DXyyX|6aRC-_%474e7w#!Tm(!l#vv=_{i!2H zK~_29x*@7Ls%Xu78UD}^j2~Lg(PtaNbL=TfHZBsm=r*yB?ABlp1DsN6a*6bdgU}Tq z4CffEY-@c{<{H!P)MozxdZ5%_ef-38^P1|Wv^grjc&S#+gsW~PZU?1Z(Vr_R;+=z> z4oxyf>;|pDV>HY#yGN^1Z6cIgDi}p{cGh;bgEC>?)rCu$N|$p16%~JpTy!I)bqH+c zZM$Lz-D-Ku_FiQh9Y#YGYxc!fRv1}0{c7~OO|*lEV?OBEsctMioRP6Uqk7hD9FZ)e zAAK7lxu(#Kq%>#b?E06ClTGGaCSm?9@=2qK*Oz`Zh zB!!UV8qR2OI&r|{=9n6@ER-5xGy#X=AaD&o!~__9>fl6k%ol$3shO#`dJ4=Je)IvSxqSs@8_aj04NdFlD<|!~^bFVz zJ*y+_-t+;fG*Dco+2TnT(bO8wTio}c1fRoNJmBuKMgIVBmHt%ry~6dSVcnT;;C&pN z$M%Si`x6?!_V0Qug6z!y0E8mkf2!24Q`;zin5{v5??sSmX0L=@KIW@8{0cQc%N3{g z?|LkRQC|&QuT4o$)(`&5uYI_B(PSEsN1ZL@=mS+x`$X1oy!W6DKtS$K)Yfm?J?H~Pq#3MIq3=K!^*IOjUFZFAfBgj3Hlcr=I_PJtFZ_)FeDtT~6pDBi1(GSGS^$1& zGe8$DP88EEkH~4FWKPXdlT;#TkIhlW+|UL6-z`F0<)p@mt=s3Qt=s3J36?J3E-DMB z`NbiV*u*Zb)X9f9$j0sjEktg2My(Qb56qt5R3I!k+J zwvB$rD3T@MA?Fn}otwYUWhNK0X1LpI^emQvEzOjOMaU7#D806~bv}BLDEq*Q5NKd( zFCI^oaIU%FFzHOWv$F)@@meBTlO(fbmDol)3Ytq>aDG{8%$g{)({fn>;~Z1vfGNUF z)~J@5OL+~Qqyy}t71%w7gT))nr;OD+;MC@fKIWs=MNRtj24Cm7|w-T!V0C)1>(sQ`uPRCiN-UFsc6lJ`#l^;sO zg6u^O6@c~!wPe<&`WYI&l@+rq$M?FveN9WK&So6vpIXu_qZy>gq5-YKo|wieubNRZ zynSk{Mafp$6eY5_suDcXE>r-ZXL0RL#N4?x8$Cq9*w*6hR*diJMeCnhY{tnJ1? zrv068dCgXYd&bs)hF2u2$QPwMM+pf#w+{aREoC!mXx)++i2dcO4|Sm~20wQ>2hr;@ za+J_Qf7QzIpGK<_yRiN2D<`_sO_I_wu54OJ&+lV@uUdS`180_u`c)=FNX41EJ0%<9I4Z}etYr%n zozA;(3QlU3ysQq%VA_2_;-bYI_Op7@&Qq|dJV9t8m&`$cYiL-`S1}7zWY(r+o@on; PfSzaocBZl7n2-P27pt*X literal 0 HcmV?d00001 diff --git a/modules/modeldata.py b/modules/modeldata.py index 51cbc05df..b9b9a28dd 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -50,6 +50,8 @@ def get_model_type(pipe): model_type = 'h1' elif "Cosmos2TextToImage" in name: model_type = 'cosmos' + elif "FLite" in name: + model_type = 'flite' # video models elif "CogVideo" in name: model_type = 'cogvideo' diff --git a/modules/sd_detect.py b/modules/sd_detect.py index 5fdc268fd..5555751ee 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -95,6 +95,8 @@ def guess_by_name(fn, current_guess): return 'FLEX' elif 'cosmos-predict2' in fn.lower(): return 'Cosmos' + elif 'f-lite' in fn.lower(): + return 'FLite' return current_guess diff --git a/modules/sd_models.py b/modules/sd_models.py index 011d75c6f..c69a612a0 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -91,7 +91,7 @@ def set_vae_options(sd_model, vae=None, op:str='model', quiet:bool=False): sd_model.enable_vae_slicing() else: sd_model.disable_vae_slicing() - if hasattr(sd_model, "enable_vae_tiling"): + if hasattr(sd_model, "enable_vae_tiling") and hasattr(sd_model, "disable_vae_tiling"): if shared.opts.diffusers_vae_tiling: if hasattr(sd_model, 'vae') and hasattr(sd_model.vae, 'config') and hasattr(sd_model.vae.config, 'sample_size') and isinstance(sd_model.vae.config.sample_size, int): if getattr(sd_model.vae, "tile_sample_min_size_backup", None) is None: @@ -361,6 +361,10 @@ def load_diffuser_force(model_type, checkpoint_info, diffusers_load_config, op=' from pipelines.model_cosmos import load_cosmos_t2i sd_model = load_cosmos_t2i(checkpoint_info, diffusers_load_config) allow_post_quant = False + elif model_type in ['FLite']: + from pipelines.model_flite import load_flite + sd_model = load_flite(checkpoint_info, diffusers_load_config) + allow_post_quant = False except Exception as e: shared.log.error(f'Load {op}: path="{checkpoint_info.path}" {e}') if debug_load: diff --git a/modules/shared_items.py b/modules/shared_items.py index 1cd8f8cb9..8dc2f19ca 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -51,6 +51,7 @@ pipelines = { 'OmniGen2': getattr(diffusers, 'DiffusionPipeline', None), # dynamically redefined and loaded in sd_models.load_diffuser 'InstaFlow': getattr(diffusers, 'DiffusionPipeline', None), # dynamically redefined and loaded in sd_models.load_diffuser 'SegMoE': getattr(diffusers, 'DiffusionPipeline', None), # dynamically redefined and loaded in sd_models.load_diffuser + 'FLite': getattr(diffusers, 'DiffusionPipeline', None), } initialize_onnx() diff --git a/pipelines/f_lite/__init__.py b/pipelines/f_lite/__init__.py new file mode 100644 index 000000000..e4ddb20f5 --- /dev/null +++ b/pipelines/f_lite/__init__.py @@ -0,0 +1,5 @@ +from .pipeline import FLitePipeline, FLitePipelineOutput, APGConfig +from .model import DiT + + +__all__ = ["APGConfig", "DiT", "FLitePipeline", "FLitePipelineOutput"] diff --git a/pipelines/f_lite/f_lite.model.py b/pipelines/f_lite/f_lite.model.py new file mode 100644 index 000000000..e56e6545b --- /dev/null +++ b/pipelines/f_lite/f_lite.model.py @@ -0,0 +1,455 @@ +# DiT with cross attention + +import math + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import FromOriginalModelMixin, PeftAdapterMixin +from diffusers.models.modeling_utils import ModelMixin +from diffusers.utils.accelerate_utils import apply_forward_hook +from einops import rearrange +from peft import get_peft_model_state_dict, set_peft_model_state_dict +from torch import nn + + +def timestep_embedding(t, dim, max_period=10000): + half = dim // 2 + freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).to( + device=t.device + ) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + + return embedding + + +class RMSNorm(nn.Module): + def __init__(self, dim, eps=1e-6, trainable=False): + super().__init__() + self.eps = eps + if trainable: + self.weight = nn.Parameter(torch.ones(dim)) + else: + self.weight = None + + def forward(self, x): + x_dtype = x.dtype + x = x.float() + norm = torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + if self.weight is not None: + return (x * norm * self.weight).to(dtype=x_dtype) + else: + return (x * norm).to(dtype=x_dtype) + + +class QKNorm(nn.Module): + """Normalizing the query and the key independently, as Flux proposes""" + + def __init__(self, dim, trainable=False): + super().__init__() + self.query_norm = RMSNorm(dim, trainable=trainable) + self.key_norm = RMSNorm(dim, trainable=trainable) + + def forward(self, q, k): + q = self.query_norm(q) + k = self.key_norm(k) + return q, k + + +class Attention(nn.Module): + def __init__( + self, + dim, + num_heads=8, + qkv_bias=False, + is_self_attn=True, + cross_attn_input_size=None, + residual_v=False, + dynamic_softmax_temperature=False, + ): + super().__init__() + assert dim % num_heads == 0 + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = self.head_dim**-0.5 + self.is_self_attn = is_self_attn + self.residual_v = residual_v + self.dynamic_softmax_temperature = dynamic_softmax_temperature + + if is_self_attn: + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + else: + self.q = nn.Linear(dim, dim, bias=qkv_bias) + self.context_kv = nn.Linear(cross_attn_input_size, dim * 2, bias=qkv_bias) + + self.proj = nn.Linear(dim, dim, bias=False) + + if residual_v: + self.lambda_param = nn.Parameter(torch.tensor(0.5).reshape(1)) + + self.qk_norm = QKNorm(self.head_dim) + + def forward(self, x, context=None, v_0=None, rope=None): + if self.is_self_attn: + qkv = self.qkv(x) + qkv = rearrange(qkv, "b l (k h d) -> k b h l d", k=3, h=self.num_heads) + q, k, v = qkv.unbind(0) + + if self.residual_v and v_0 is not None: + v = self.lambda_param * v + (1 - self.lambda_param) * v_0 + + if rope is not None: + # print(q.shape, rope[0].shape, rope[1].shape) + q = apply_rotary_emb(q, rope[0], rope[1]) + k = apply_rotary_emb(k, rope[0], rope[1]) + + # https://arxiv.org/abs/2306.08645 + # https://arxiv.org/abs/2410.01104 + # ratioonale is that if tokens get larger, categorical distribution get more uniform + # so you want to enlargen entropy. + + token_length = q.shape[2] + if self.dynamic_softmax_temperature: + ratio = math.sqrt(math.log(token_length) / math.log(1040.0)) # 1024 + 16 + k = k * ratio + q, k = self.qk_norm(q, k) + + else: + q = rearrange(self.q(x), "b l (h d) -> b h l d", h=self.num_heads) + kv = rearrange( + self.context_kv(context), + "b l (k h d) -> k b h l d", + k=2, + h=self.num_heads, + ) + k, v = kv.unbind(0) + q, k = self.qk_norm(q, k) + + x = F.scaled_dot_product_attention(q, k, v) + x = rearrange(x, "b h l d -> b l (h d)") + x = self.proj(x) + return x, v if self.is_self_attn else None + + +class DiTBlock(nn.Module): + def __init__( + self, + hidden_size, + cross_attn_input_size, + num_heads, + mlp_ratio=4.0, + qkv_bias=True, + residual_v=False, + dynamic_softmax_temperature=False, + ): + super().__init__() + self.hidden_size = hidden_size + self.norm1 = RMSNorm(hidden_size, trainable=qkv_bias) + self.self_attn = Attention( + hidden_size, + num_heads=num_heads, + qkv_bias=qkv_bias, + is_self_attn=True, + residual_v=residual_v, + dynamic_softmax_temperature=dynamic_softmax_temperature, + ) + + if cross_attn_input_size is not None: + self.norm2 = RMSNorm(hidden_size, trainable=qkv_bias) + self.cross_attn = Attention( + hidden_size, + num_heads=num_heads, + qkv_bias=qkv_bias, + is_self_attn=False, + cross_attn_input_size=cross_attn_input_size, + dynamic_softmax_temperature=dynamic_softmax_temperature, + ) + else: + self.norm2 = None + self.cross_attn = None + + self.norm3 = RMSNorm(hidden_size, trainable=qkv_bias) + mlp_hidden = int(hidden_size * mlp_ratio) + self.mlp = nn.Sequential( + nn.Linear(hidden_size, mlp_hidden), + nn.GELU(), + nn.Linear(mlp_hidden, hidden_size), + ) + + self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 9 * hidden_size, bias=True)) + + self.adaLN_modulation[-1].weight.data.zero_() + self.adaLN_modulation[-1].bias.data.zero_() + + # @torch.compile(mode='reduce-overhead') + def forward(self, x, context, c, v_0=None, rope=None): + ( + shift_sa, + scale_sa, + gate_sa, + shift_ca, + scale_ca, + gate_ca, + shift_mlp, + scale_mlp, + gate_mlp, + ) = self.adaLN_modulation(c).chunk(9, dim=1) + + scale_sa = scale_sa[:, None, :] + scale_ca = scale_ca[:, None, :] + scale_mlp = scale_mlp[:, None, :] + + shift_sa = shift_sa[:, None, :] + shift_ca = shift_ca[:, None, :] + shift_mlp = shift_mlp[:, None, :] + + gate_sa = gate_sa[:, None, :] + gate_ca = gate_ca[:, None, :] + gate_mlp = gate_mlp[:, None, :] + + norm_x = self.norm1(x.clone()) + norm_x = norm_x * (1 + scale_sa) + shift_sa + attn_out, v = self.self_attn(norm_x, v_0=v_0, rope=rope) + x = x + attn_out * gate_sa + + if self.norm2 is not None: + norm_x = self.norm2(x) + norm_x = norm_x * (1 + scale_ca) + shift_ca + x = x + self.cross_attn(norm_x, context)[0] * gate_ca + + norm_x = self.norm3(x) + norm_x = norm_x * (1 + scale_mlp) + shift_mlp + x = x + self.mlp(norm_x) * gate_mlp + + return x, v + + +class PatchEmbed(nn.Module): + def __init__(self, patch_size=16, in_channels=3, embed_dim=768): + super().__init__() + self.patch_proj = nn.Conv2d(in_channels, embed_dim, kernel_size=patch_size, stride=patch_size) + self.patch_size = patch_size + + def forward(self, x): + B, C, H, W = x.shape + x = self.patch_proj(x) + x = rearrange(x, "b c h w -> b (h w) c") + return x + + +class TwoDimRotary(torch.nn.Module): + def __init__(self, dim, base=10000, h=256, w=256): + super().__init__() + self.inv_freq = torch.FloatTensor([1.0 / (base ** (i / dim)) for i in range(0, dim, 2)]) + self.h = h + self.w = w + + t_h = torch.arange(h, dtype=torch.float32) + t_w = torch.arange(w, dtype=torch.float32) + + freqs_h = torch.outer(t_h, self.inv_freq).unsqueeze(1) # h, 1, d / 2 + freqs_w = torch.outer(t_w, self.inv_freq).unsqueeze(0) # 1, w, d / 2 + freqs_h = freqs_h.repeat(1, w, 1) # h, w, d / 2 + freqs_w = freqs_w.repeat(h, 1, 1) # h, w, d / 2 + freqs_hw = torch.cat([freqs_h, freqs_w], 2) # h, w, d + + self.register_buffer("freqs_hw_cos", freqs_hw.cos()) + self.register_buffer("freqs_hw_sin", freqs_hw.sin()) + + def forward(self, x, height_width=None, extend_with_register_tokens=0): + if height_width is not None: + this_h, this_w = height_width + else: + this_hw = x.shape[1] + this_h, this_w = int(this_hw**0.5), int(this_hw**0.5) + + cos = self.freqs_hw_cos[0 : this_h, 0 : this_w] + sin = self.freqs_hw_sin[0 : this_h, 0 : this_w] + + cos = cos.clone().reshape(this_h * this_w, -1) + sin = sin.clone().reshape(this_h * this_w, -1) + + # append N of zero-attn tokens + if extend_with_register_tokens > 0: + cos = torch.cat( + [ + torch.ones(extend_with_register_tokens, cos.shape[1]).to(cos.device), + cos, + ], + 0, + ) + sin = torch.cat( + [ + torch.zeros(extend_with_register_tokens, sin.shape[1]).to(sin.device), + sin, + ], + 0, + ) + + return cos[None, None, :, :], sin[None, None, :, :] # [1, 1, T + N, Attn-dim] + + +def apply_rotary_emb(x, cos, sin): + orig_dtype = x.dtype + x = x.to(dtype=torch.float32) + assert x.ndim == 4 # multihead attention + d = x.shape[3] // 2 + x1 = x[..., :d] + x2 = x[..., d:] + y1 = x1 * cos + x2 * sin + y2 = x1 * (-sin) + x2 * cos + return torch.cat([y1, y2], 3).to(dtype=orig_dtype) + + +class DiT(ModelMixin, ConfigMixin, FromOriginalModelMixin, PeftAdapterMixin): # type: ignore[misc] + @register_to_config + def __init__( + self, + in_channels=4, + patch_size=2, + hidden_size=1152, + depth=28, + num_heads=16, + mlp_ratio=4.0, + cross_attn_input_size=128, + residual_v=False, + train_bias_and_rms=True, + use_rope=True, + gradient_checkpoint=False, + dynamic_softmax_temperature=False, + rope_base=10000, + ): + super().__init__() + + self.patch_embed = PatchEmbed(patch_size, in_channels, hidden_size) + + if use_rope: + self.rope = TwoDimRotary(hidden_size // (2 * num_heads), base=rope_base, h=512, w=512) + else: + self.positional_embedding = nn.Parameter(torch.zeros(1, 2048, hidden_size)) + + self.register_tokens = nn.Parameter(torch.randn(1, 16, hidden_size)) + + self.time_embed = nn.Sequential( + nn.Linear(hidden_size, 4 * hidden_size), + nn.SiLU(), + nn.Linear(4 * hidden_size, hidden_size), + ) + + self.blocks = nn.ModuleList( + [ + DiTBlock( + hidden_size=hidden_size, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + cross_attn_input_size=cross_attn_input_size, + residual_v=residual_v, + qkv_bias=train_bias_and_rms, + dynamic_softmax_temperature=dynamic_softmax_temperature, + ) + for _ in range(depth) + ] + ) + + self.final_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True)) + + self.final_norm = RMSNorm(hidden_size, trainable=train_bias_and_rms) + self.final_proj = nn.Linear(hidden_size, patch_size * patch_size * in_channels) + nn.init.zeros_(self.final_modulation[-1].weight) + nn.init.zeros_(self.final_modulation[-1].bias) + nn.init.zeros_(self.final_proj.weight) + nn.init.zeros_(self.final_proj.bias) + self.paramstatus = {} + for n, p in self.named_parameters(): + self.paramstatus[n] = { + "shape": p.shape, + "requires_grad": p.requires_grad, + } + + def save_lora_weights(self, save_directory): + """Save LoRA weights to a file""" + lora_state_dict = get_peft_model_state_dict(self) + torch.save(lora_state_dict, f"{save_directory}/lora_weights.pt") + + def load_lora_weights(self, load_directory): + """Load LoRA weights from a file""" + lora_state_dict = torch.load(f"{load_directory}/lora_weights.pt") + set_peft_model_state_dict(self, lora_state_dict) + + @apply_forward_hook + def forward(self, x, context, timesteps): + b, c, h, w = x.shape + x = self.patch_embed(x) # b, T, d + + x = torch.cat([self.register_tokens.repeat(b, 1, 1), x], 1) # b, T + N, d + + if self.config.use_rope: + cos, sin = self.rope( + x, + extend_with_register_tokens=16, + height_width=(h // self.config.patch_size, w // self.config.patch_size), + ) + else: + x = x + self.positional_embedding.repeat(b, 1, 1)[:, : x.shape[1], :] + cos, sin = None, None + + t_emb = timestep_embedding(timesteps * 1000, self.config.hidden_size).to(x.device, dtype=x.dtype) + t_emb = self.time_embed(t_emb) + + v_0 = None + + for _idx, block in enumerate(self.blocks): + if self.config.gradient_checkpoint: + x, v = torch.utils.checkpoint.checkpoint( + block, + x, + context, + t_emb, + v_0, + (cos, sin), + use_reentrant=False, + ) + else: + x, v = block(x, context, t_emb, v_0, (cos, sin)) + if v_0 is None: + v_0 = v + + x = x[:, 16:, :] + final_shift, final_scale = self.final_modulation(t_emb).chunk(2, dim=1) + x = self.final_norm(x) + x = x * (1 + final_scale[:, None, :]) + final_shift[:, None, :] + x = self.final_proj(x) + + x = rearrange( + x, + "b (h w) (p1 p2 c) -> b c (h p1) (w p2)", + h=h // self.config.patch_size, + w=w // self.config.patch_size, + p1=self.config.patch_size, + p2=self.config.patch_size, + ) + return x + + +if __name__ == "__main__": + model = DiT( + in_channels=4, + patch_size=2, + hidden_size=1152, + depth=28, + num_heads=16, + mlp_ratio=4.0, + cross_attn_input_size=128, + residual_v=False, + train_bias_and_rms=True, + use_rope=True, + ).cuda() + print( + model( + torch.randn(1, 4, 64, 64).cuda(), + torch.randn(1, 37, 128).cuda(), + torch.tensor([1.0]).cuda(), + ) + ) diff --git a/pipelines/f_lite/model.py b/pipelines/f_lite/model.py new file mode 100644 index 000000000..e56e6545b --- /dev/null +++ b/pipelines/f_lite/model.py @@ -0,0 +1,455 @@ +# DiT with cross attention + +import math + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import FromOriginalModelMixin, PeftAdapterMixin +from diffusers.models.modeling_utils import ModelMixin +from diffusers.utils.accelerate_utils import apply_forward_hook +from einops import rearrange +from peft import get_peft_model_state_dict, set_peft_model_state_dict +from torch import nn + + +def timestep_embedding(t, dim, max_period=10000): + half = dim // 2 + freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).to( + device=t.device + ) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + + return embedding + + +class RMSNorm(nn.Module): + def __init__(self, dim, eps=1e-6, trainable=False): + super().__init__() + self.eps = eps + if trainable: + self.weight = nn.Parameter(torch.ones(dim)) + else: + self.weight = None + + def forward(self, x): + x_dtype = x.dtype + x = x.float() + norm = torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + if self.weight is not None: + return (x * norm * self.weight).to(dtype=x_dtype) + else: + return (x * norm).to(dtype=x_dtype) + + +class QKNorm(nn.Module): + """Normalizing the query and the key independently, as Flux proposes""" + + def __init__(self, dim, trainable=False): + super().__init__() + self.query_norm = RMSNorm(dim, trainable=trainable) + self.key_norm = RMSNorm(dim, trainable=trainable) + + def forward(self, q, k): + q = self.query_norm(q) + k = self.key_norm(k) + return q, k + + +class Attention(nn.Module): + def __init__( + self, + dim, + num_heads=8, + qkv_bias=False, + is_self_attn=True, + cross_attn_input_size=None, + residual_v=False, + dynamic_softmax_temperature=False, + ): + super().__init__() + assert dim % num_heads == 0 + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = self.head_dim**-0.5 + self.is_self_attn = is_self_attn + self.residual_v = residual_v + self.dynamic_softmax_temperature = dynamic_softmax_temperature + + if is_self_attn: + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + else: + self.q = nn.Linear(dim, dim, bias=qkv_bias) + self.context_kv = nn.Linear(cross_attn_input_size, dim * 2, bias=qkv_bias) + + self.proj = nn.Linear(dim, dim, bias=False) + + if residual_v: + self.lambda_param = nn.Parameter(torch.tensor(0.5).reshape(1)) + + self.qk_norm = QKNorm(self.head_dim) + + def forward(self, x, context=None, v_0=None, rope=None): + if self.is_self_attn: + qkv = self.qkv(x) + qkv = rearrange(qkv, "b l (k h d) -> k b h l d", k=3, h=self.num_heads) + q, k, v = qkv.unbind(0) + + if self.residual_v and v_0 is not None: + v = self.lambda_param * v + (1 - self.lambda_param) * v_0 + + if rope is not None: + # print(q.shape, rope[0].shape, rope[1].shape) + q = apply_rotary_emb(q, rope[0], rope[1]) + k = apply_rotary_emb(k, rope[0], rope[1]) + + # https://arxiv.org/abs/2306.08645 + # https://arxiv.org/abs/2410.01104 + # ratioonale is that if tokens get larger, categorical distribution get more uniform + # so you want to enlargen entropy. + + token_length = q.shape[2] + if self.dynamic_softmax_temperature: + ratio = math.sqrt(math.log(token_length) / math.log(1040.0)) # 1024 + 16 + k = k * ratio + q, k = self.qk_norm(q, k) + + else: + q = rearrange(self.q(x), "b l (h d) -> b h l d", h=self.num_heads) + kv = rearrange( + self.context_kv(context), + "b l (k h d) -> k b h l d", + k=2, + h=self.num_heads, + ) + k, v = kv.unbind(0) + q, k = self.qk_norm(q, k) + + x = F.scaled_dot_product_attention(q, k, v) + x = rearrange(x, "b h l d -> b l (h d)") + x = self.proj(x) + return x, v if self.is_self_attn else None + + +class DiTBlock(nn.Module): + def __init__( + self, + hidden_size, + cross_attn_input_size, + num_heads, + mlp_ratio=4.0, + qkv_bias=True, + residual_v=False, + dynamic_softmax_temperature=False, + ): + super().__init__() + self.hidden_size = hidden_size + self.norm1 = RMSNorm(hidden_size, trainable=qkv_bias) + self.self_attn = Attention( + hidden_size, + num_heads=num_heads, + qkv_bias=qkv_bias, + is_self_attn=True, + residual_v=residual_v, + dynamic_softmax_temperature=dynamic_softmax_temperature, + ) + + if cross_attn_input_size is not None: + self.norm2 = RMSNorm(hidden_size, trainable=qkv_bias) + self.cross_attn = Attention( + hidden_size, + num_heads=num_heads, + qkv_bias=qkv_bias, + is_self_attn=False, + cross_attn_input_size=cross_attn_input_size, + dynamic_softmax_temperature=dynamic_softmax_temperature, + ) + else: + self.norm2 = None + self.cross_attn = None + + self.norm3 = RMSNorm(hidden_size, trainable=qkv_bias) + mlp_hidden = int(hidden_size * mlp_ratio) + self.mlp = nn.Sequential( + nn.Linear(hidden_size, mlp_hidden), + nn.GELU(), + nn.Linear(mlp_hidden, hidden_size), + ) + + self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 9 * hidden_size, bias=True)) + + self.adaLN_modulation[-1].weight.data.zero_() + self.adaLN_modulation[-1].bias.data.zero_() + + # @torch.compile(mode='reduce-overhead') + def forward(self, x, context, c, v_0=None, rope=None): + ( + shift_sa, + scale_sa, + gate_sa, + shift_ca, + scale_ca, + gate_ca, + shift_mlp, + scale_mlp, + gate_mlp, + ) = self.adaLN_modulation(c).chunk(9, dim=1) + + scale_sa = scale_sa[:, None, :] + scale_ca = scale_ca[:, None, :] + scale_mlp = scale_mlp[:, None, :] + + shift_sa = shift_sa[:, None, :] + shift_ca = shift_ca[:, None, :] + shift_mlp = shift_mlp[:, None, :] + + gate_sa = gate_sa[:, None, :] + gate_ca = gate_ca[:, None, :] + gate_mlp = gate_mlp[:, None, :] + + norm_x = self.norm1(x.clone()) + norm_x = norm_x * (1 + scale_sa) + shift_sa + attn_out, v = self.self_attn(norm_x, v_0=v_0, rope=rope) + x = x + attn_out * gate_sa + + if self.norm2 is not None: + norm_x = self.norm2(x) + norm_x = norm_x * (1 + scale_ca) + shift_ca + x = x + self.cross_attn(norm_x, context)[0] * gate_ca + + norm_x = self.norm3(x) + norm_x = norm_x * (1 + scale_mlp) + shift_mlp + x = x + self.mlp(norm_x) * gate_mlp + + return x, v + + +class PatchEmbed(nn.Module): + def __init__(self, patch_size=16, in_channels=3, embed_dim=768): + super().__init__() + self.patch_proj = nn.Conv2d(in_channels, embed_dim, kernel_size=patch_size, stride=patch_size) + self.patch_size = patch_size + + def forward(self, x): + B, C, H, W = x.shape + x = self.patch_proj(x) + x = rearrange(x, "b c h w -> b (h w) c") + return x + + +class TwoDimRotary(torch.nn.Module): + def __init__(self, dim, base=10000, h=256, w=256): + super().__init__() + self.inv_freq = torch.FloatTensor([1.0 / (base ** (i / dim)) for i in range(0, dim, 2)]) + self.h = h + self.w = w + + t_h = torch.arange(h, dtype=torch.float32) + t_w = torch.arange(w, dtype=torch.float32) + + freqs_h = torch.outer(t_h, self.inv_freq).unsqueeze(1) # h, 1, d / 2 + freqs_w = torch.outer(t_w, self.inv_freq).unsqueeze(0) # 1, w, d / 2 + freqs_h = freqs_h.repeat(1, w, 1) # h, w, d / 2 + freqs_w = freqs_w.repeat(h, 1, 1) # h, w, d / 2 + freqs_hw = torch.cat([freqs_h, freqs_w], 2) # h, w, d + + self.register_buffer("freqs_hw_cos", freqs_hw.cos()) + self.register_buffer("freqs_hw_sin", freqs_hw.sin()) + + def forward(self, x, height_width=None, extend_with_register_tokens=0): + if height_width is not None: + this_h, this_w = height_width + else: + this_hw = x.shape[1] + this_h, this_w = int(this_hw**0.5), int(this_hw**0.5) + + cos = self.freqs_hw_cos[0 : this_h, 0 : this_w] + sin = self.freqs_hw_sin[0 : this_h, 0 : this_w] + + cos = cos.clone().reshape(this_h * this_w, -1) + sin = sin.clone().reshape(this_h * this_w, -1) + + # append N of zero-attn tokens + if extend_with_register_tokens > 0: + cos = torch.cat( + [ + torch.ones(extend_with_register_tokens, cos.shape[1]).to(cos.device), + cos, + ], + 0, + ) + sin = torch.cat( + [ + torch.zeros(extend_with_register_tokens, sin.shape[1]).to(sin.device), + sin, + ], + 0, + ) + + return cos[None, None, :, :], sin[None, None, :, :] # [1, 1, T + N, Attn-dim] + + +def apply_rotary_emb(x, cos, sin): + orig_dtype = x.dtype + x = x.to(dtype=torch.float32) + assert x.ndim == 4 # multihead attention + d = x.shape[3] // 2 + x1 = x[..., :d] + x2 = x[..., d:] + y1 = x1 * cos + x2 * sin + y2 = x1 * (-sin) + x2 * cos + return torch.cat([y1, y2], 3).to(dtype=orig_dtype) + + +class DiT(ModelMixin, ConfigMixin, FromOriginalModelMixin, PeftAdapterMixin): # type: ignore[misc] + @register_to_config + def __init__( + self, + in_channels=4, + patch_size=2, + hidden_size=1152, + depth=28, + num_heads=16, + mlp_ratio=4.0, + cross_attn_input_size=128, + residual_v=False, + train_bias_and_rms=True, + use_rope=True, + gradient_checkpoint=False, + dynamic_softmax_temperature=False, + rope_base=10000, + ): + super().__init__() + + self.patch_embed = PatchEmbed(patch_size, in_channels, hidden_size) + + if use_rope: + self.rope = TwoDimRotary(hidden_size // (2 * num_heads), base=rope_base, h=512, w=512) + else: + self.positional_embedding = nn.Parameter(torch.zeros(1, 2048, hidden_size)) + + self.register_tokens = nn.Parameter(torch.randn(1, 16, hidden_size)) + + self.time_embed = nn.Sequential( + nn.Linear(hidden_size, 4 * hidden_size), + nn.SiLU(), + nn.Linear(4 * hidden_size, hidden_size), + ) + + self.blocks = nn.ModuleList( + [ + DiTBlock( + hidden_size=hidden_size, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + cross_attn_input_size=cross_attn_input_size, + residual_v=residual_v, + qkv_bias=train_bias_and_rms, + dynamic_softmax_temperature=dynamic_softmax_temperature, + ) + for _ in range(depth) + ] + ) + + self.final_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True)) + + self.final_norm = RMSNorm(hidden_size, trainable=train_bias_and_rms) + self.final_proj = nn.Linear(hidden_size, patch_size * patch_size * in_channels) + nn.init.zeros_(self.final_modulation[-1].weight) + nn.init.zeros_(self.final_modulation[-1].bias) + nn.init.zeros_(self.final_proj.weight) + nn.init.zeros_(self.final_proj.bias) + self.paramstatus = {} + for n, p in self.named_parameters(): + self.paramstatus[n] = { + "shape": p.shape, + "requires_grad": p.requires_grad, + } + + def save_lora_weights(self, save_directory): + """Save LoRA weights to a file""" + lora_state_dict = get_peft_model_state_dict(self) + torch.save(lora_state_dict, f"{save_directory}/lora_weights.pt") + + def load_lora_weights(self, load_directory): + """Load LoRA weights from a file""" + lora_state_dict = torch.load(f"{load_directory}/lora_weights.pt") + set_peft_model_state_dict(self, lora_state_dict) + + @apply_forward_hook + def forward(self, x, context, timesteps): + b, c, h, w = x.shape + x = self.patch_embed(x) # b, T, d + + x = torch.cat([self.register_tokens.repeat(b, 1, 1), x], 1) # b, T + N, d + + if self.config.use_rope: + cos, sin = self.rope( + x, + extend_with_register_tokens=16, + height_width=(h // self.config.patch_size, w // self.config.patch_size), + ) + else: + x = x + self.positional_embedding.repeat(b, 1, 1)[:, : x.shape[1], :] + cos, sin = None, None + + t_emb = timestep_embedding(timesteps * 1000, self.config.hidden_size).to(x.device, dtype=x.dtype) + t_emb = self.time_embed(t_emb) + + v_0 = None + + for _idx, block in enumerate(self.blocks): + if self.config.gradient_checkpoint: + x, v = torch.utils.checkpoint.checkpoint( + block, + x, + context, + t_emb, + v_0, + (cos, sin), + use_reentrant=False, + ) + else: + x, v = block(x, context, t_emb, v_0, (cos, sin)) + if v_0 is None: + v_0 = v + + x = x[:, 16:, :] + final_shift, final_scale = self.final_modulation(t_emb).chunk(2, dim=1) + x = self.final_norm(x) + x = x * (1 + final_scale[:, None, :]) + final_shift[:, None, :] + x = self.final_proj(x) + + x = rearrange( + x, + "b (h w) (p1 p2 c) -> b c (h p1) (w p2)", + h=h // self.config.patch_size, + w=w // self.config.patch_size, + p1=self.config.patch_size, + p2=self.config.patch_size, + ) + return x + + +if __name__ == "__main__": + model = DiT( + in_channels=4, + patch_size=2, + hidden_size=1152, + depth=28, + num_heads=16, + mlp_ratio=4.0, + cross_attn_input_size=128, + residual_v=False, + train_bias_and_rms=True, + use_rope=True, + ).cuda() + print( + model( + torch.randn(1, 4, 64, 64).cuda(), + torch.randn(1, 37, 128).cuda(), + torch.tensor([1.0]).cuda(), + ) + ) diff --git a/pipelines/f_lite/pipeline.py b/pipelines/f_lite/pipeline.py new file mode 100644 index 000000000..18b327087 --- /dev/null +++ b/pipelines/f_lite/pipeline.py @@ -0,0 +1,303 @@ +import logging +import math +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +from diffusers import AutoencoderKL, DiffusionPipeline +from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor +from PIL import Image +from torch import FloatTensor +from tqdm.auto import tqdm +from transformers import T5EncoderModel, T5TokenizerFast + + + +logger = logging.getLogger(__name__) + + +@dataclass +class APGConfig: + """APG (Augmented Parallel Guidance) configuration""" + + enabled: bool = True + orthogonal_threshold: float = 0.03 + + +@dataclass +class FLitePipelineOutput(BaseOutput): + """ + Output class for FLitePipeline pipeline. + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. + """ + + images: Union[List[Image.Image], np.ndarray] + + +class FLitePipeline(DiffusionPipeline): + r""" + Pipeline for text-to-image generation using F-Lite model. + This model inherits from [`DiffusionPipeline`]. + """ + + model_cpu_offload_seq = "text_encoder->dit_model->vae" + + dit_model: torch.nn.Module + vae: AutoencoderKL + text_encoder: T5EncoderModel + tokenizer: T5TokenizerFast + _progress_bar_config: Dict[str, Any] + + def __init__( + self, dit_model: torch.nn.Module, vae: AutoencoderKL, text_encoder: T5EncoderModel, tokenizer: T5TokenizerFast + ): + super().__init__() + # Register all modules for the pipeline + # Access DiffusionPipeline's register_modules directly to avoid mypy error + DiffusionPipeline.register_modules( + self, dit_model=dit_model, vae=vae, text_encoder=text_encoder, tokenizer=tokenizer + ) + + # Move models to channels last for better performance + # AutoencoderKL inherits from torch.nn.Module which has these methods + if hasattr(self.vae, "to"): + self.vae.to(memory_format=torch.channels_last) + if hasattr(self.vae, "requires_grad_"): + self.vae.requires_grad_(False) + if hasattr(self.text_encoder, "requires_grad_"): + self.text_encoder.requires_grad_(False) + + # Constants + self.vae_scale_factor = 8 + self.return_index = -8 # T5 hidden state index to use + + def enable_vae_slicing(self): + """Enable VAE slicing for memory efficiency.""" + if hasattr(self.vae, "enable_slicing"): + self.vae.enable_slicing() + + def enable_vae_tiling(self): + """Enable VAE tiling for memory efficiency.""" + if hasattr(self.vae, "enable_tiling"): + self.vae.enable_tiling() + + def set_progress_bar_config(self, **kwargs): + """Set progress bar configuration.""" + self._progress_bar_config = kwargs + + def progress_bar(self, iterable=None, **kwargs): + """Create progress bar for iterations.""" + self._progress_bar_config = getattr(self, "_progress_bar_config", None) or {} + config = {**self._progress_bar_config, **kwargs} + return tqdm(iterable, **config) + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + max_sequence_length: int = 512, + return_index: int = -8, + ) -> Tuple[FloatTensor, FloatTensor]: + """Encodes the prompt and negative prompt.""" + if isinstance(prompt, str): + prompt = [prompt] + device = self._execution_device + # Text encoder forward pass + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids.to(device) + prompt_embeds = self.text_encoder(text_input_ids, return_dict=True, output_hidden_states=True) + prompt_embeds_tensor = prompt_embeds.hidden_states[return_index] + if return_index != -1: + prompt_embeds_tensor = self.text_encoder.encoder.final_layer_norm(prompt_embeds_tensor) + prompt_embeds_tensor = self.text_encoder.encoder.dropout(prompt_embeds_tensor) + + dtype = dtype or next(self.text_encoder.parameters()).dtype + prompt_embeds_tensor = prompt_embeds_tensor.to(dtype=dtype, device=device) + + # Handle negative prompts + if negative_prompt is None: + negative_embeds = torch.zeros_like(prompt_embeds_tensor) + else: + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + negative_result = self.encode_prompt( + prompt=negative_prompt, device=device, dtype=dtype, return_index=return_index + ) + negative_embeds = negative_result[0] + + # Explicitly cast both tensors to FloatTensor for mypy + from typing import cast + + prompt_tensor = cast("FloatTensor", prompt_embeds_tensor.to(dtype=dtype)) + negative_tensor = cast("FloatTensor", negative_embeds.to(dtype=dtype)) + return (prompt_tensor, negative_tensor) + + def to(self, torch_device=None, torch_dtype=None, silence_dtype_warnings=False): + """Move pipeline components to specified device and dtype.""" + if hasattr(self, "vae"): + self.vae.to(device=torch_device, dtype=torch_dtype) + if hasattr(self, "text_encoder"): + self.text_encoder.to(device=torch_device, dtype=torch_dtype) + if hasattr(self, "dit_model"): + self.dit_model.to(device=torch_device, dtype=torch_dtype) + return self + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]], + height: Optional[int] = 1024, + width: Optional[int] = 1024, + num_inference_steps: int = 30, + guidance_scale: float = 6.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + dtype: Optional[torch.dtype] = None, + alpha: Optional[float] = None, + apg_config: Optional[APGConfig] = None, + **kwargs, + ): + """Generate images from text prompt.""" + # Ensure height and width are not None for calculation + if height is None: + height = 1024 + if width is None: + width = 1024 + + dtype = dtype or next(self.dit_model.parameters()).dtype + apg_config = apg_config or APGConfig(enabled=False) + + device = self._execution_device + + # 2. Encode prompts + prompt_batch_size = len(prompt) if isinstance(prompt, list) else 1 + batch_size = prompt_batch_size * num_images_per_prompt + + prompt_embeds, negative_embeds = self.encode_prompt( + prompt=prompt, negative_prompt=negative_prompt, device=device, dtype=dtype, + return_index=self.return_index, + ) + + # Repeat embeddings for num_images_per_prompt + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + negative_embeds = negative_embeds.repeat_interleave(num_images_per_prompt, dim=0) + + # 3. Initialize latents + latent_height = height // self.vae_scale_factor + latent_width = width // self.vae_scale_factor + + if isinstance(generator, list): + if len(generator) != batch_size: + raise ValueError(f"Got {len(generator)} generators for {batch_size} samples") + + latents = randn_tensor((batch_size, 16, latent_height, latent_width), generator=generator, device=device, dtype=dtype) + acc_latents = latents.clone() + + # 4. Calculate alpha if not provided + if alpha is None: + image_token_size = latent_height * latent_width + alpha = 2 * math.sqrt(image_token_size / (64 * 64)) + + # 6. Sampling loop + self.dit_model.eval() + + # Check if guidance is needed + do_classifier_free_guidance = guidance_scale >= 1.0 + + for i in self.progress_bar(range(num_inference_steps, 0, -1)): + # Calculate timesteps + t = i / num_inference_steps + t_next = (i - 1) / num_inference_steps + # Scale timesteps according to alpha + t = t * alpha / (1 + (alpha - 1) * t) + t_next = t_next * alpha / (1 + (alpha - 1) * t_next) + dt = t - t_next + + # Create tensor with proper device + t_tensor = torch.tensor([t] * batch_size, device=device, dtype=dtype) + + if do_classifier_free_guidance: + # Duplicate latents for both conditional and unconditional inputs + latents_input = torch.cat([latents] * 2) + # Concatenate negative and positive prompt embeddings + context_input = torch.cat([negative_embeds, prompt_embeds]) + # Duplicate timesteps for the batch + t_input = torch.cat([t_tensor] * 2) + + # Get model predictions in a single pass + model_outputs = self.dit_model(latents_input, context_input, t_input) + + # Split outputs back into unconditional and conditional predictions + uncond_output, cond_output = model_outputs.chunk(2) + + if apg_config.enabled: + # Augmented Parallel Guidance + dy = cond_output + dd = cond_output - uncond_output + # Find parallel direction + parallel_direction = (dy * dd).sum() / (dy * dy).sum() * dy + orthogonal_direction = dd - parallel_direction + # Scale orthogonal component + orthogonal_std = orthogonal_direction.std() + orthogonal_scale = min(1, apg_config.orthogonal_threshold / orthogonal_std) + orthogonal_direction = orthogonal_direction * orthogonal_scale + model_output = dy + (guidance_scale - 1) * orthogonal_direction + else: + # Standard classifier-free guidance + model_output = uncond_output + guidance_scale * (cond_output - uncond_output) + else: + # If no guidance needed, just run the model normally + model_output = self.dit_model(latents, prompt_embeds, t_tensor) + + # Update latents + acc_latents = acc_latents + dt * model_output.to(device) + latents = acc_latents.clone() + + # 7. Decode latents + # These checks handle the case where mypy doesn't recognize these attributes + scaling_factor = getattr(self.vae.config, "scaling_factor", 0.18215) if hasattr(self.vae, "config") else 0.18215 + shift_factor = getattr(self.vae.config, "shift_factor", 0) if hasattr(self.vae, "config") else 0 + + latents = latents / scaling_factor + shift_factor + + vae_dtype = self.vae.dtype if hasattr(self.vae, "dtype") else dtype + decoded_images = self.vae.decode(latents.to(vae_dtype)).sample if hasattr(self.vae, "decode") else latents + + # Offload all models + try: + self.maybe_free_model_hooks() + except AttributeError as e: + if "OptimizedModule" in str(e): + import warnings + warnings.warn( + "Encountered 'OptimizedModule' error when offloading models. " + "This issue might be fixed in the future by: " + "https://github.com/huggingface/diffusers/pull/10730", + stacklevel=1, + ) + else: + raise + + # 8. Post-process images + images = (decoded_images / 2 + 0.5).clamp(0, 1) + # Convert to PIL Images + images = (images * 255).round().clamp(0, 255).to(torch.uint8).cpu() + pil_images = [Image.fromarray(img.permute(1, 2, 0).numpy()) for img in images] + + return FLitePipelineOutput( + images=pil_images, + ) diff --git a/pipelines/model_cosmos.py b/pipelines/model_cosmos.py index c9fad08db..4bdd95ef3 100644 --- a/pipelines/model_cosmos.py +++ b/pipelines/model_cosmos.py @@ -47,11 +47,6 @@ def load_text_encoder(repo_id, diffusers_load_config={}): ) if shared.opts.diffusers_offload_mode != 'none' and text_encoder is not None: sd_models.move_model(text_encoder, devices.cpu) - - load_args, quant_args = model_quant.get_dit_args(diffusers_load_config, module='TE', device_map=True) - llama_repo = shared.opts.model_h1_llama_repo if shared.opts.model_h1_llama_repo != 'Default' else 'meta-llama/Meta-Llama-3.1-8B-Instruct' - shared.log.debug(f'Load model: type=HiDream te4="{llama_repo}" quant="{model_quant.get_quant_type(quant_args)}" args={load_args}') - return text_encoder diff --git a/pipelines/model_flite.py b/pipelines/model_flite.py new file mode 100644 index 000000000..9c1426fb4 --- /dev/null +++ b/pipelines/model_flite.py @@ -0,0 +1,63 @@ +import sys +import transformers +from modules import shared, devices, sd_models, model_quant, sd_hijack_te + + +def load_dit(repo_id, diffusers_load_config={}): + load_args, quant_args = model_quant.get_dit_args(diffusers_load_config, module='Model', device_map=True) + shared.log.debug(f'Load model: type=FLite dit="{repo_id}" quant="{model_quant.get_quant_type(quant_args)}" args={load_args}') + import pipelines.f_lite + sys.modules['f_lite'] = pipelines.f_lite + transformer = pipelines.f_lite.DiT.from_pretrained( + repo_id, + subfolder="dit_model", + cache_dir=shared.opts.hfcache_dir, + **load_args, + **quant_args, + ) + if shared.opts.diffusers_offload_mode != 'none' and transformer is not None: + sd_models.move_model(transformer, devices.cpu) + return transformer + + +def load_text_encoder(repo_id, diffusers_load_config={}): + load_args, quant_args = model_quant.get_dit_args(diffusers_load_config, module='TE', device_map=True) + shared.log.debug(f'Load model: type=FLite te="{repo_id}" quant="{model_quant.get_quant_type(quant_args)}" args={load_args}') + text_encoder = transformers.T5EncoderModel.from_pretrained( + repo_id, + subfolder="text_encoder", + cache_dir=shared.opts.hfcache_dir, + **load_args, + **quant_args, + ) + if shared.opts.diffusers_offload_mode != 'none' and text_encoder is not None: + sd_models.move_model(text_encoder, devices.cpu) + return text_encoder + + +def load_flite(checkpoint_info, diffusers_load_config={}): + repo_id = sd_models.path_to_repo(checkpoint_info) + sd_models.hf_auth_check(checkpoint_info) + + from pipelines.f_lite import FLitePipeline + dit_model = load_dit(repo_id, diffusers_load_config) + text_encoder = load_text_encoder(repo_id, diffusers_load_config) + + load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, module='Model') + shared.log.debug(f'Load model: type=FLite model="{checkpoint_info.name}" repo="{repo_id}" offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}') + pipe = FLitePipeline.from_pretrained( + repo_id, + revision="refs/pr/8", + dit_model=dit_model, + text_encoder=text_encoder, + trust_remote_code=True, + cache_dir=shared.opts.diffusers_dir, + **load_args, + ) + + sd_hijack_te.init_hijack(pipe) + del text_encoder + del dit_model + + devices.torch_gc() + return pipe