From 7c16ea6b1da29ce06f5af1c5cc74d233ba3c66df Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 5 Dec 2023 11:27:45 -0500 Subject: [PATCH] add modelscope --- CHANGELOG.md | 2 ++ html/reference.json | 5 +++++ .../damo-vilab--text-to-video-ms-1.7b.jpg | Bin 0 -> 18687 bytes modules/processing_diffusers.py | 10 ++++++++-- scripts/animatediff.py | 2 ++ scripts/stablevideodiffusion.py | 12 ++++++++---- 6 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 models/Reference/damo-vilab--text-to-video-ms-1.7b.jpg diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e57de16e..481507267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - **Diffusers** - **IP Adapter** add support for `ip-adapter-plus_sd15` and `ip-adapter-plus-face_sd15` + - added support for basic [ModelScope T2V](https://huggingface.co/damo-vilab/text-to-video-ms-1.7b) model + - simply select from *networks -> reference* and use from *txt2img* tab - **General** - **LoRA** add support for block weights, thanks @AI-Casanova example `` diff --git a/html/reference.json b/html/reference.json index a3ff809ed..44a4b15e2 100644 --- a/html/reference.json +++ b/html/reference.json @@ -88,5 +88,10 @@ "path": "thu-ml/unidiffuser-v1", "desc": "UniDiffuser is a unified diffusion framework to fit all distributions relevant to a set of multi-modal data in one transformer. UniDiffuser is able to perform image, text, text-to-image, image-to-text, and image-text pair generation by setting proper timesteps without additional overhead.\nSpecifically, UniDiffuser employs a variation of transformer, called U-ViT, which parameterizes the joint noise prediction network. Other components perform as encoders and decoders of different modalities, including a pretrained image autoencoder from Stable Diffusion, a pretrained image ViT-B/32 CLIP encoder, a pretrained text ViT-L CLIP encoder, and a GPT-2 text decoder finetuned by ourselves.", "preview": "thu-ml--unidiffuser-v1.jpg" + }, + "ModelScope T2V": { + "path": "damo-vilab/text-to-video-ms-1.7b", + "desc": "The text-to-video generation diffusion model consists of three sub-networks: text feature extraction model, text feature-to-video latent space diffusion model, and video latent space to video visual space model. The overall model parameters are about 1.7 billion. Currently, it only supports English input. The diffusion model adopts a UNet3D structure, and implements video generation through the iterative denoising process from the pure Gaussian noise video.", + "preview": "damo-vilab--text-to-video-ms-1.7b.jpg" } } \ No newline at end of file diff --git a/models/Reference/damo-vilab--text-to-video-ms-1.7b.jpg b/models/Reference/damo-vilab--text-to-video-ms-1.7b.jpg new file mode 100644 index 0000000000000000000000000000000000000000..39ad83eea52da9122edccbafef77ba0b6d47f6fb GIT binary patch literal 18687 zcmbTdc|25q{69KUX_Jo@N--5u*(zIEW-1kurc#8MimWlo&X`jvDl{R4GL^L~Q(*`DA-!38$N%R;{2d{?K>}{4q{KGF8d(WRSqX6i0tvT#9XuxghUNb} zB-X%fTrag@Uo^xPNeOuJBxMn31j`)wBSuW7i4hdA z!xk?_aM8@zp5>_b+MMn$b;v<*Qbp$mt*tFXj0h9GIEAEbvoc_+nr9M)DS2qmmpE-I zkq2Q$-*55@A+dikTbEIwo#7Lp8XN${h>D9+G%F!VYaG0*BRq5RJxCW&MWL^@q*1F< z(b}NBHL4H!P89s_(aRS%pgYBgAWq1BG2#JcnU*O=n28Zdq$Qd3hZ^+^F`}zF(V?9P z61j)gP~Z8A5f{Y>ni%2v0s2!WMx;NV5+lBzlEX6vx5Nk&XTTE@f`CY(i+_22ndl3I zh7J~^OVN|bOf<6+(G~S85~-!VS|vuXXgNZnDDa_vZ4Z3kns6~<1UH(Wdk+Lwj+}CD zlM$6HBO!@vP%Bzg3=RUTU90b%DGL!TcIf~ZC`J^fK{f(w*&GIe0owM@k`h$tgKUUF zJtdEx%S=q;*ohGX$zUa-6M{$C7W$lL%HY#kOo*ORw(@@SQX5pv4cO8}fY8g3S>tVu1Fa1NCo21JWIG_316SyxYu_s- z#0aWYoEQ;;riL~_)NcH97d||cB43c+`$|EK=ocf(#Ry^!d*bt#@Is_u1`;FYPAQmy z7|}A?4<6c0Su(V#Ct^f?0#A$(tj3*>`ZvIi=$FO+<^&!XAvPA_1Ty~s^y?TQO2`Fx z;BiZM@ZmlUK$XBwQ;-xUta}}=D@Gi6s1Y9D2f8H-e@*5<-OH2^N~=(eP(AQ62ebgY zvD_iQq}0eyG9nI}D}vls8rK+PM?fI(6{Oo2h!NgE6d>C4{=FEH4e;Rmv7)qB5a$8G z3^#BEI&#X}%1!FsUKXFez}K!?;(XRObnCY`ZX|OHO)fa9kDiQ5Mhd9oAcLKM66)X( zhv1O_svvIfLye(Xk_$)|E%VU>p1;M23u&N>lA9Re^lA@wh$#;wFq-8FJH>q?s9boaFixHZK*7O0Zz>;ABRP{@YXjXZ@WgroL9LnM2 zT0R7faRj}LzsmnXY?L)mU#2thVMHd%Ev*^U%@iU&B9nZCZ zVo$w3a9~~71Ydo@5g#}2_M5t)8+{iSIlgPDM6&N*(H1dcyl1Yj|FYBt#S^XmC|7*V zII`d2J10;1Z(T?Z6FTJck}e9BPkwRf-&eg$)2UY7^?0r7do6W0Z zd2%oIl~fJ==kWZtH+|-^#;@VV8_3=-P+LNO^6KL!3aqNMlfN+_b+plkhvjrv7UPr~ zkXuCZ1r0TU9zSTk+(U+{p&LZ`cZO0*22&xoeoT z92kv!L2wlVp(yRNbMnS>PO9-1)l}}xD;m&>=D;AKDAWFE{Qa!5ZySMq)fpP^whmdo;9M2~ zgIt&hDv`J&ZG%~La^PbbSx4)Fr06Bf1YGtEHX{XTe5T^q$ z#uxfid2iaFkLJK2^jgB;U#AVa_AjG*sA7a$z9s7c2AllK&Nr9t6XCbp-HQI?Xle(9HWhfM240R{n7S;_OTgJY@%AiNS z78AAT&9w-k>wPY5tf2r( z;ReWCJ(LbDw6w@z#LDqG>Nan5Bt=*8YRE{ES8(ku;hrh!87D3aI2CQvsg3qmO*+dz zC>#_j(2h@AqE7|{E;D7AR^Jy?NTHRO?XCmY%FAlIoR@9mw0bZ01qw2hAN~u4zq%{C zGE&{fzj=NOJUDbAa0spNQfWo?{xr^mTjcP~kg@VPX$VmGk=@v^a^v2l&GBspL4}b` zvM7>NVpppoVHIz3z4gQF+4d+nD0~B!JHG)zYe?Xy%N)J)7y)zG0?9siK0HJu7c$V{M!il9V<9r`XDx7Ys^BVK&cugMv3NYGt+$k_Ww zudtCc&xh%kopy2!C4zaKYSb=9T!9GhJfH1{e!RpDwG;v1CXxXs>K*h|4PNhwd=JDo znsMko^V<=v#{q)rMd(lb?hLS=Dh0JIT$Hksf%h6RNJb4jMrU`Ut5TVu*&v1~Mi?Rv zb}gWL&5S#m#fX=_W}v?qVIxKui4m)x5HKJe{s7};Iu;JZq1lmp%z>WoBs#>rfcRLc zSkRZUi`YtE&hD4-EE7@%jEkEu&j5F$pb$|SfN$paczq2wSR5}+hJ;9{@|2oj$5An2 zZTvZizM9XR51aV_GPt#qM|E9$bmzwR>4#a60%zkT~AiiPqMW6fT>qn?x@eh_R{ndX;v5*q2l@sv_-x^-} z_*GBSd#lWN-?ISnUc5WYI9(?lyI^8)IjMbf2&IR1w(yLKMb{!512j?2qut548^w{m}F#bUAIQ;f1i0 z?f;ZYlRQh=b?geKYyX+`e$0y9cbRAH=31GlGaTN-*mN8NG{0!-VtfM9a&MDe8e_C$;D5O2j25=LL&VZxZKWztE5xJesdXOp}zcD(G5j z49RkgAr{<&^rtCuAKQjvLY=A!D`B-}+)JzMCSrhUde&N2%xiq#)x_WP;b_jTg|(Rv zws^1R?$e1?&3Y?>Y*3~7E`MKM-3TwQj48r0y};j3dC#5zTX+%MUNrmicgQoIRyTRB zIjQR8oC8Y!jTqtDPZ&p6Vu#%(XRxv_ZsxDvI_e?Fq6>H;3Qy5LFGMljxURFja!-b& zPzrtuHUMI9DGEKpn?b;GGbC{;#-qeT6AtxuaE$m0dR+=f^+EP#y72(z)GoS9!UDF6 z!~>Y*tb6=p5EGR8oiGKV6Wt}%TG+SQBAC$6UGj3kt|s({kQSQ3Swsssc$EJDquIwb zsHOF${fvMdVNy@w-f!UohPUnlpVEGp7GX%BE0t-}n9~n1D9Hza2d{SBA2MrB$p+-P zK=hmgX}TC_E*NTN;YmS$VWHNNsc${x6J&#&cl&#j{1$t;|H9u0CjC)%Z==khDMaQ1 z3t~j6b$I+qO0VUq2;o(Lk5j=8g7o15Gmc}74v-h3($A*oBB(|!%PKE{VW5nQK9a@_ zB(bSH6k|AZoe@SGbc0UX5c;J=#r;zUhrq*++!ii}NBm|Rw`^a$2Mdf3;D;On8?y?l zls@zrr*WSO)E|%IGo*ww6_g9tS?ga9DPF~J5+<|PewmGHlSu-YlQz&`7bjwU%P=N+ z!n!E);{+{7v;t0${3iGwBwAeCB*bimrHx4fGyKr5ZwgK5pox$~4Fgu&77|1qH8-8= zQAtOx;y6!)r9tgra|*mB#0bSqDKFtsm;xRga0mu2>nUqi@@97g&o# zrg#M;ixB|{{H!ptNKSHew6m0s2>z@iY|Zmn>Cb?snzoq<%)5_w@EJKjB@@w&T}z+y zI*Z&c+I=|qscN5E>+0t|5^G6}h+^kndC{-%D|0yZRm*-?jpl=IS}IFyP7b(SgW2Ep z!S?zi_F8h=1|Sryfr4!)Pga(5y^i|qdVE0jl^9VZvubSfRRz&41&-Yd6|Y}vQ&9w8o(;2Kc?I+$_lNBAJD!!*P|$s=H!4Ad%-Z|z z)y-8gVoRGmNBTqjNuWEhKzp+o@l&n|T?{dQK_K2ohZmJ+d1uhtVQ8F?;)~=GfEwF9 z*Iw)bf`rt?9QuG0j%KhKUCf7N_5`c-%utI#7gS!lUk)fiheLTFb@4jRh=m$LptnGtT0HLO&WUjp!(P{#3m*Gjz`1VfpDgm1eFEofde(JyQ z+CC5)r{@8N!HE1`LFv+UT$w*eE3Bm?yeL44&1XF}- z%>-(45buSHVR*8TfL#)GfA4~*M92VbSg2U|c;Hws^Ro7luhfrp^L#mcVG?DIj3*O84a-1p+hvK*+gt z&7K8+P^P^bS7>NoUqKl;%aN?myAqvHB>5M(YvNF!F3+Rf1_~XU?;LmzwTaXjI|ryf ziA?kwG2(Wc@(jD*h%>~VemRg>NKM&TBc4qF@$BsQ6_ zu(us&ID*8?q#a;E*Bi}IW9_U(mx}B% z|H2#&_2=fLaWP{5BSBBtZhsf>j^Z7Bi{FN)kz3D&lSirjQ9o+;CQ}WsGoo<-Bl{yh z`U=#SRGl8~pwv_Oys;hsZj=<6%6PHXuhC2OTXGT%EU`a!MobFn1}xe12(=mPZC)97 zJ$fNJFL*)I+QsI}g89DFqC?RSGyZP1!YR03GK=Q9jZPNE`|ZVX%n2^b_SivZY}grnJ*(nljE4=q~dgbm%-7vp!1OeWftf?PvSP-79$!u7jeTd z$>*S!!^029-vbMKk&D3B8vP#ac|N>fZK1{H_nUK^9{0LK6C?vsar0`vQO$a^UG9c-kXSw19rq3gzD9N**GBD|fja1=q$lmJJd9TEkwaoHMw z7PjKjP>ww*&ZEM#TnmG3ZN-d!k<}@ssHLb8D2wIQXp!m8x`!sr&agWMn!%}6j$+$Q zFdEYKOyA1mRi2xq&7@xE+$w)D9?L^&D^-xaWFB9QDsj`8)tx6q;gxxsqtu8>?Y|{n z*p&+O_!NX&?E*1(SFU_7Eack1q?(u4Kx0YBC^6!5)$4$-iEV}qvhg|DFOTP-)B}{U zKrzBT(B($I5X%&i`Ces!Xl<`mKAoj2Mi35Sgs|pbo%U?pLt(>_MgmG)_hi_=LJ_rK zc4cK>uLkOGHIR>~s0q@pJzWrb3m?r&vu5EQc~8YRj~WK&VPZt_jk#p##XVV`ZMQ;G zH12c4)OEF7f3&KxqtHIM!!^kW6J?Wn7#Y3K%z86uha|y&#JGuh)?ak*KST5ZwLsrfV za>*U&eyLH^^K3D~hZk|JW30DOG_A2ReegPNINtG%+oNXdzmgVxj*_qVX`{*W5##h{ zZSsFhLNw=(0$o0EYn2E&1 z3eFU$b>nkIH&{Utiy~x!rQ3aDSY|#R{}tScxxaS}S{8eW@F(dTw6FSYYuaNh=qxIz z%2`=qIwdN!y52q%Xxv;M7z8Z6q&-vlXOOMQ4t4;|;;4(jz*Mc}(QJMKRH>Ax_JIs0 z2e>_VtHB6Y(Cr@F1T)xSMX#W1oOEo=P8%es34Ss^h!I{n>}*q$WYL#W_A~+x@|npV zTn-LM%2>`x10CZnq_}9ZF03|-U=4bDZ3g!;v^@V@O}Ils?Hy87Rtft_^N<`j```Bc z83mF#u>EfM3&#(~@T6L+mU6&C?IAFIg9{%$nT0R|lNmr>mAM$f!OdBl1_c-_ipl}{ z`?XMOQ3T{v-M^hFB4wf6z2RmMT+(V8T+j2@D*w!rvn?jlgW(07Z917_r7K6NB;sM~ zeHxp!=16!lWEXucftSX}Lwmtj;3k<6e~8zEXth#8bCmf(_vn|~a$19ereIHPX+7)@ zl)v9h>E|LjR48Pi_1mZL_%Wow4dgOb^44_I2%ij+cPLp^0}Hi_=%)8k3zzx8GL`2p zd4`H%1ASCjko?TO9f=tRsF+cH?W(JtJx*&6d8h?i!8Y2W>h?hs4*-BiF+}t(5t4Ix z12?-$9D=t~E934E+k_S9Uj-tg?gD5xueZ5@d)ex9z@~%6u#;k6@!LrpsaSdnA)kmw;i!|)p@?W7o<4<~eWUUIXM;L}b%}dFm2Yc?3 z)~T!XZeE;x*=<1MFc$g5xvqVi1ix;av~(dFUcWln0)9bhPorf~M?3^MPxG@nVS~7~ zHNgLa?eE0BBZY$xYIKHcfh8b9beg5z7+9;gv`Es=$sM+sJg2Y)ypjGrKqGY1>fv9z z0)9PcF-#>D0L5iZGqw7JHVQ5IFVy`Md#SRSb>VrpoA7(Ys-Edc((_EbJkL#x*w7xQ zS(xSh!Ia^MHUDclPz-w=dw?P1t$%k;T%UYU9cVZn<{9e9c=5i=_*kyPkmcmNjzP0M z;mow#kPCF3ozsYkwYASK*do~3fho>7-)XfRk}}5+EVNaL-G=Ina9HoXZF)&dtC}R7 zk;bPiHFF|h<1bpc{S5B7SB-(qH*H_F^N4vreyXR%yd0{o$M~#2d9&kSc*cBD%l(dM zBA>DBtP6(yI2PRcLr@rP98Wvv&(?k?KYTO&eMi83)cq=64M=2dlIk+H_RY0ZshaSL zE*K^-M`Z)%n#{5ugmOjO3n83rtPMuV6q!UK*lv1dmiKq)#!xkGqNXdh z$N+<7z=~${{G5^st3;vqT`qC3%J8Pqx1E?1&1~qJ#0*Im?cQeeeK{nML;SpM%vx@0 zpdCdjz$dUe$E}ybo2qhltH}q+R=cC_c|)tWCV!meB`Rv*acUo2al|?OGr|Pf#p*Wj0BeTkwWNZ-Hs$ z)xw;SXQgSTm6fH(`C&;&E!@Zqigb&<%DAOZ8TYLgipnk*GF8HHXySDA7*9e=mz zO^{sQEIRvtUorZ|IANV2+P6mjJ=rOJV|L-n$t*knqyCQP4f@p7p1hibeBQ1Um>6Ip z7LI&>^V``-J|lP55R|Ev_8b3F^jcRd_FT9X)vy;3cz{FC9sX;ZaQ+D~OY!T>AGBK2 zVhUeiXMj1C5|F;cCxv)U9Hk0dRMU|{=4AfENE{j}zsH9~g|v3R7KH(eOc^@AoU`M5 z9>dZA41nE15z7n~ag%QW3LU--7F)pqJ09YX(yX9j*e)>Qb?bN{S}hF8x9Zm+<;w+I$hv*C0kPd$0>u z==t=UOCi9LCY;qUm=}=V34dye$eamb;-#)e!@JePH~-URoAsrquY9Kv;Vr=TkO7PN z19S67$Rbse7p!}NnCQWQQqPBK@csa1FmOk?pE(J+gI?+Ty8P+$yoR9UD-Qg4XWpB` z;NixTM*HuD#|ii2Hx_+eI>q`{a~I6Y#rk;b_fF&E&KjDVfE;*X+9?i8|xOV55>wGJB}YNK`&Ts zZ3;WA>t3kB7DWj`;v5v(w{Z-oiNo}ty|cl$Y?4MOFWeOG<$TQ<>v{e>p?;-Mt*I7O zv{XP!SBV@u74qEa>*NEc8M6*xGhQllCNz=sJYW09?)(qkHp=@Oh*OUahX0Xxv}kpU z@VoX{R1hVSafhGU^ck1$re^7dNtiVu zzY(HG7VeA_8>KH z0~ib@ixKWt+s9mqfhw%6X<8qkviZx<8`!frLDKkU)+^QH!qXF+*UFY_v+e1i>V5-Y_P$VrJfd&Rrln}?aR*&D&6rG z0Y9pe4AH-C&m;XhO7@KIt^05;_R8hLABq=Vo%eg__wclo@FT1P>zp(yDtluL0>^%x zr)`2=$bEu9G^56ybxln^H}Pb4m`@rR$g=R{6y6eMvDkv1HZ*u-?=I+6x$rN!{cG`) z&t=;Oa(;ib3i7y{OISQ9D(_l6V2$=QNSXW|*8;+f(ki5B0{a(cg+)Nan3K?Pe0AZ9 z7~$4pASx4*>MpJ~eGg4xc&XvAC6x|@4U>XA;OZEtPi+Y=^pJ?tj-0>_f!Sw(na6}u zvWG4N+o>S{nz}5aK%Z}E#^wur(o~LBM*zbhV^{awTtQI6ffB7OyMtAp}H#i5riF1h(;d#5 zg&-T@6t(;AWl9W6*GsPUWH&@rt**RVQ-3C@=zD(FwednpztzH^akZ!WmMBDR?*&{Z zOG}34AncImf71)sP~PLHoJhLPI~a~`MD6M-4cBUWVrv151k@~=r9I;7B)^1f&N%g_ z{hjLHl1NM9PzmYyB5+}X9EH2_#DxfJhawp5=akv-9=vtU(DlQE{b>V4~yTYaiYVT#wsnZU|`>@))!s&&h4LgxsuySa^O%uw5DJEmVuHXSiB z6>9Y=dVzcA@Nc3}yJ+# zQCaEE_^}#$=9t$6ehfI4LKj@$V`LQT{__ss}~xp2=!&9S!{ID_5FRoAGh* z++-^b>@z_wL66IcQX*9rCbl%s(dc4VW_sB39r#%2zLb4vWYj=v*)#3JRpNHQ#d2-N z^{##HDx|5G;qCHo;i(zy%b!LUww@)L`(6bU z?T4sB>0UaI`{r$#F#Ds`mQ~zi1*dl-Bn6orblAFfFRccumUpSU6*5D)GEwNYCB$b= zTZ(%9W1Sb`487xZk+XYcQWQh;WSmxdzVioebis1)$%{pS3ES*GeV^Vu_CJ(t+bDJj z6FMHl$A*;W`o3CxDR0VJM|`TemrmV=1IIdV6;c@7^%a!Z=zr`0C% z0%>wPOkKh!eC+-T68KwWU&H{Z?0(kyB_PZpcJ-LBR5-0Zp_)CM!~@)LQtE2X>^?4=rxGaqFsK}b0O(p=C!QIH)!jf>;EueqIJ0hKLm zRH4+@9&s^z0_Ta)+R(boSB!gu{QRcQ!Ay~Ap`0CFB!trmwv#|b?L&3P;Lu~Rvupxk z-f^x|4Vgkz@B|UgDEPd2rdpcR4BlLF1j^yMiKKeW@(NGwv5XGy&>G|l*$Bj@KyW;i z_OQz0AHW=kKloFxB{0xIB)ik{1`I)AeuV4xUMJyvc@b-t%ulIv9<2hdjVt*_Xm2PRvGimG40J>*iG62*j>DGcdo*Y z7S}m7rqY=HZmVY}M-C&O;!8&dcCSC%nii54INU!lcT+Fx5-qQ|GIpcI+()!owb3~F zVW{IuAaiW40?!;xU9-bCen@@9tIcGpIAZgoD$r|h{IQtuV3AkSJM+f59saEwvtvN- zt_hy$c&vg-arc6n%DdtaQkdW{`Ik@;u2HOyOZo6_RQgTUmhoNVlC;trWj(SQGVGti zv~k-QV+WzGL+z&#>h3nJH2tC8t!q2BlA=y>4%F`IGxjcY^ysn=>7=^Pe)#Yzi3!6G z-63S9z&ZNCwCmDw#|!8ZgT^#sOu9yCUuIWEmk;y6+46qw!B@W3;a(3}Pn!h+B{eT# z+s%zFQZN0*kka<0Exa%5$bTqYYSTRlhdd0gODiTX6XHUI%15GeepXF_ZCSs9=*7)- z3w~qVXA@m3(Z~|nGJiU|edj8gv~$&LUjE}S@17&po|52;N8KIZJ1?!hItSM!cEFAq z#WhU(sXab!rX#%A?iD{s;cW}MaY3Q)3-IG?zy@y{&0c}_yHYZ0K3)qQ+%_#SuG@G2 zkmvAvwzlqJ{J^IJ1|8DrfjxtTTW5~d3H;YoW8w9;;P@dpv|UrXK6LPb1) zhQBjx7#%qk{&k!g#Ys1e_?3>;!X}HxD97>F#fWuQEUnVq@Fb()`~}Dy7;|J(gl4d@ zW!!U~BfF|i1n%VGtxOsMr~397@N@st=*dc@jL>lIivaRgN2i&xAX}Dayb3cwb=mM6 z;W{?K$ksTlbMo|qV8E3cjg&YT(V(=pCc=>jIcehQ=~t$0GXTQ{ZDTLi=K3VV^U-%T zQg3pWfXwiZ=69cCzaaG&a=IVAIH^~@s13tf4!IdnoC~VmSEhe_da=Ma<0g>5yr&xo zGEr|a23W05JZ<5rV6`S%j4aT=HIEI*yMdjE+l)*bddLVeX}qE~ZSyHWKFZ%tr7S|} zB38u>R-A&oM#3SeXZXLR<|zNL0)s1MJ#b~vPK9Az+=x%b3fdmg(+M3oozDxpQc_#X zE@KA8{g)p*A5r--g^B$S>Le)yNV_%krj|uzmzS}9Wl{`=VMwhs3Yfc=aUT`YOHc{M z{@@g+Ii442nj$Mxag~(QsXRYO&L(f{@S>KUuMlP|&U@fgdtQ)a882<@r090;h2`P4 zPgZ?x75u#42U&7Mrd{m|_UycPJzbHPY*V=vb;_tu(`!sUk|mR(LaEXeeV(;0+7>s* zy*Xx~jn7c`+o*$m-#zUD5BzXk*xc=~gE27R01^ zU3+bnYwE?J49OhvhcT8Eu;o6qXr%reb{bl7Gi?Zm9vRw7FjFOg_pPk0BAAm~2*DGw z0brti4>A+n+StOLcm;!GkSLXcPAv8VXD3G^F{^BhFqU4&{v&0Y1%90MP=N)y$MQi@ zFNu>0xCpO32uXvmRqTR&`gpA&wYk3%wxWs{)0q*iA==qAt>Nz)3Y zt$)Ii>AJF}U7J(_92r(gi`JA9Rk)nEI8lsXe7r1~Bp9B{_X8@5y%pZU4#~YmH64gA z`|SV_wnKeI!FcmH@`|b-U~LYXm&}qm2lYkNJ6&+f zd6O*9-RQF%0k!In$dvCK+R)>me52=y@-J$V|H8?8V}tLHA+;v=Y75aH$R7w=gx}~L z#(Uf};Ka(-QV9>A579GToYx6tbQSfIq3~Kc=CjfMXE7wvc82shXV=HWst5*x@hBmN zaHmX{{sjHs?RWnLTROyh<>zAO`=b)OBAOMCR4C|i;udesV%_xp>|UL|x`{hSok{2| z2reFtimgUZIXd$Eu0H&LBh9~jU~)++SMN$TSG6ailc5TJHRxAUc?#N$Cfy4L7|6hNvvt$08XX zQ7tmygyteV&t}7L4!hX?g1)LzI!kEJ6E>u;TfV*2TcBY`__}rOXLf7IRV9?oUYX>0 z6JJLz5!y8%3jL$oY@l@2bh%O`&HCko;qqDacLkPcBcIaKpO3{fzUi8BqaLpb?Q0@G zwLwW%%F%8@_lO=FOwn%X5Y@-!YFvwK<9} zAE&Fu>#;?1pUTt)Lc;az$EZVnwI-gI=i;4eXU;K1B`GN>pRq6KGApruHU@-UUG#I0 zHXr}ns%%bL+UC9@Db=iGSjI|vH;P?Q4mnx;(!qOt>hANR`rRwg_1{)tMM&=HNe8|x zE$8*;18`Wwod!J~_0N}-xi2h0J;4NPITW+E3{sd)y>P<|?Oj<`R$IDxvY_<{|5>iE z8TDqTt!&Ud*D%4JHa30@j;6U*RYjQ6@UKEc-W(@%33!-~KOHnKv!$+}RY^$wHY=OS zLI<$>%Lt~ak@HP>`k3}3V5v~F34d-Z>`501^eBCE_l=O-Qi$aI1~XT1MD#XP>!_x~ zLhYVT=mzk}#|1ik^wckbt<@2q?5PLIhsO*fYrw36aPlwNb8sFs13A>_>eBah70`*M z$@L66hQz&vu}R_Dv6i3B>xs8-FAuIzw#;?Z-S*fy@wHB)^B(bcIOP%f04wN?;A}!; zLtIgdlaG$%#U(;v2+Dc%4YzG(c zCbAO06n|X&W@6r5F`Mk?Xf;XW%PR)q=gO7}4p+kxCnWv?3l8RZxMKF^!3f`=kA(%<#KSBW`!tNc%*>l@N z((i<@@$+ev3l*G!{1E6YG9FsUMZ6gVZo{eJB%<09u&wK$!qY)|bvnHjy%0{nb!d$u z+7d3}=^R-ei@!t<0ETx6jf|@K08FLEJ|XORKC|t(!MFAn6;6_(#!H=p_80wX(Fsr} zvHi_%ZYq|U@lQ8fS544Gs6=BLW66vHReJN`#F3_M6=tZJE`s!k;8!mhP!}?QR@bH> z`|kr(Gq{9gs4U=fZ{=j6Gh2Dt8Ls~kT-NzZaiv6fu#jVVpMNVks#)Ews^(MMBG_u8 z4KK~!>#Ziw*NuEzvaL_^$a>hE>xG<2kbZi@mn{h=f*H^{r(WciQKm_|hvkPwS|}VF z&=(rItIsKt669M3z{OD8>$Yb?zMG>WQ||TYdM(UrkS`xR&fZ-X`}L2$t2PKyggop=pzo> zLYWQIcUe*|NG?|`b}%9~43+&Q1SE`L548cMR9a< z9G04&mw;xAl3=1O7ZW_Le$=5nx`jY3oMj-tIkN$R%bOCM7!S`OJI`xG?xk_-c+ z_w3&py4U%)qWo0eCyXWP!*0*u@Tc9W;j@-I(yzrC0G^~PCGpLd%U`;S3K&76wSDd7 zbDxIOqk8Ew_`6v-!gjTVtpQ!Q>!Xq_on;qDkK!whH9=;d>Ilur%i9EC>PN@AB zhJ$7v=w+{n#6%qvBl41Ya6U2D&2CBihfVr%*5W(o6YSBaqW-{xi#y!9Y917O^$llaSX6;yhK2o+YQ=stSTNFFtVU)Gg*^D>EHq_8YY2<#6XYE?JPo;xpIo|CQ_Yp* zD81Re*HN{r9uh0``PI>?aO=y_SGu*GlR7)ffK0z-nNhA@L^l3>$&$=NJ#W)`0p5`O z+}pW7yZ=I39EnFlHke_5mI=DO=`>yNb5=k?X9l`AvF8DsjylAHU6c z=XF(Xo9U#ZecS!^Xd)q&G@;%UV0A`j%Pgs%bw0WGSnWT3PsU<74@`8@)wNx1k9k4c zTVu5^CkQt=L|g3PqP>SoIY5!wkkWYS3@~a>)I+P^fi?Cp%B*T1~uQ`d}h{?o~E zoDGc&z=0@%BN%AeQB zBVM)TWz17Z|4 zCw?hAQn%pM%zZFwosvP>=`yebet*r4^Mn@^N>-f+()erpJ zEuuw!Sx@?42)h zcvJHCK%7t65@oIB7ii}Rg4guPzjGfFevi8?TKBic43oZCXzt8HYNTE>yR zSN9SYx>|7sfwZCMIR(wZ$cT_)#1KmB z;dDY`z#bSG(&fR^WVupR8~wPx-44!+YbxKG+$H|XXRe>?;8oRsvybtEUkM{};}(Va zhJ7tInKMQ@m0HuZ$KGb`)WtEPkkn`cz?o8w2_kHv8TXz1pc|Uljh)k1C}0>XL!rQT zcZvENwP{&}=mB9q*ccf;6!*4m`z zkT#%h!X@vvlQc@A6(i(_g)xuPPY0zu&k`Y+7baby@qs0obG`$fXMA(4pS*n@I21)Rr$}DAkg#u+rYz3@cyo*T9!D{m|>x&ceh4 z#tF>?-BRIM!OoR9WQk80T4%8s@H1H#9JFBZe%5bj&@}U=m2ol9TKPqqCuy~XZ#3Q< z%hbgxB-VU=Nxx$hz^8v{-CF;JyopyM^-NbYbgavEIiu={=o?pR>N=W1Bi}mgiQHAX z3n!I474iux==e4-%KJqLgM97T{DcS6?y(oxmd|+0ksLwKual=fIPW6x%e((rMQo&(9IBzTeB$OAF5rFLECzy7>O6#kN49Q?UlSTY~HBi!isiyTph~ zU-_;`dKAxLc4HlOmQJI-9?6d z)1D%z7>DMz!f(*tNI&wYJ@ykx8#IM0Ct>erIrFV!N@Wll&UM`-?{y#ET-#S2^8S0- zo{X2wdDdj1$}=MU96JX8=4T6|?*%WJt*M&MF&r+TjYb$cy(9re~oE(Lw>jh^H zHDkx$R!f4(CMF6McjXmdp!$1sNym})?LFwxLtXZmFI3!TSa=y1%Krj{^S*-1580$= zO1mDj+izqnYr4RM;H^z^tTB)^^9^8Kp|yP+lps}1MT%@ouToB-w)1M13O6^^+~muD zjh#7oR8_HfF;vo#(nx-xB->N!%h?=r^Vr`{x)-C>%1oM(?}Dzr?W{Z=kMSERz*s6N zvDRjwtKN#5q0~+Dg#Q&V1<(2?Xx?V2c1Xyav#R!u`r^GMA88(C)aTjQ zaOzs4NYpBR&Hf_&_4a`$SwDXV&*}HETo!{2o&)%qr!gnX6R+<-y&G{~IgFw07cW!3 zoR6)proFZD{>x?NNB;mVKY8<1-cCHfE;srA0N`K#s`*2>hHYp5dO_y@02_IW{Ho9P ze14Def52Du>p;c+(~fWYXD{LZ0N1L+%Fpk7pELgeAN*=WS)N}we(pw({{SETDD$KJ z=kGt@98d#4dH(<{{{VUUe6k7A>Ht8R54@JsLQ9eT&e^B?TCx2@&z_f)86Iq%8*>L2(d z&Y>ou@gKx+c|+`Wnsj#_U+}Djuus&A`u>NO&(zzm-0AX@OxGs+CYL0i>wa3wJAv^3 z0LLvxdmkA^cO3r!VU`!*-2VVt;lFBcA7!p>H4h{aE*5CM@~3rb>5jraM)=97U3kml zMxlRo{+A(7o@4WTzTVa6o+t5j)HC0RIP+g_{{Z@Dn!QET`r4vSp}xYt4zak4?NVwQg~W?-=0&zNjXvVX-HAN0&!F|C*R>7W zeuriDuRc2+G$Kc{XO{9YcJ6X=dhztEqq#-hBpZDeKT?$)-XqbiAw6U(k80{hJxzTl z?w|ANR6g7-*h9EG_O7mI*(;fHz0LWi`!D*;)hioGCjS6Ry8i%xr~6dnJ8UZ!E&l*< zYc1TI$v6IkR<)1#^lD4qW7y7z{bJKAMnW~b%y+12y{v-84-si2R@2=-<)VDik9yRq zl{F!z(c!}|uRM|p z)s(xi$s+yMAEkWh;k|m}!}_FmaBxRlj?_Y?qj5idKV$y@+A3d--v;e&?myBzRixs_ zQ-n6%(kTA`Tl8Vi*ERW_`zQX;6Z{wWrDd&0K6E`{MYZ%_!ZLV&s6W=TojYoQGxY=F z<>!lJw3j+go;2)!_HQ$E*sd?e9~ZR^XGOY}()0ZjuGGiOhdn*3G>SY|;&+7Xd_m#0 zo(C6lyOp+-Mys+#8OD80V|YKtru^&n8m=O@k95Ct{bLT`*2~>@LfrX>#o4^KlOHta zk;ifA`c_wtwW#jAL2={*=3cbrPPrumqHIb&?`LUoqMIO?~EOe~FdYo2Rx<0x5D|@3eGvXDAn@YCS zG4tBYzr6ndWd8v5>qo^4_KykQMc4&n`SDzMnB837o9KJzhqVv3cy`9_J>p5BLymi` zWc(?f>AoH4a`{{T0MW-ok+~>3pBHANgzYe}De~uTPrlPxYQ!d4Jsh z02<^dhwQe!JdfY~{rl=Gsi*${mS~ax0KRE~FO&P3JowV9TK%eNzvJmYU%U9wGH>0( y<-s4>f9t36siw8GyFUAW$)Ez$&zt>haYwW6)H3S!x@VW>b=$Lh{{UJ9fB)I|(Qfkq literal 0 HcmV?d00001 diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index e90cd0126..95b2c1c41 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -164,7 +164,12 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro decoded = taesd_vae_decode(latents=latents) # TODO validate decoded sample diffusers # decoded = validate_sample(decoded) - imgs = model.image_processor.postprocess(decoded, output_type=output_type) + if hasattr(model, 'image_processor'): + imgs = model.image_processor.postprocess(decoded, output_type=output_type) + else: + import diffusers + image_processor = diffusers.image_processor.VaeImageProcessor() + imgs = image_processor.postprocess(decoded, output_type=output_type) shared.state.job = prev_job if shared.cmd_opts.profile: t1 = time.time() @@ -345,7 +350,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro args[arg] = task_kwargs[arg] task_args = getattr(p, 'task_args', {}) for k, v in task_args.items(): - args[k] = v + if k in possible: + args[k] = v hypertile_set(p, hr=len(getattr(p, 'init_images', []))) clean = args.copy() diff --git a/scripts/animatediff.py b/scripts/animatediff.py index 4b0ef5d9c..531335761 100644 --- a/scripts/animatediff.py +++ b/scripts/animatediff.py @@ -152,6 +152,8 @@ class Script(scripts.Script): p.extra_generation_params['AnimateDiff Lora'] = f'{lora}:{strength}' p.extra_generation_params['AnimateDiff'] = loaded_adapter p.do_not_save_grid = True + if 'animatediff' not in p.ops: + p.ops.append('animatediff') p.task_args['num_frames'] = frames p.task_args['num_inference_steps'] = p.steps if not latent_mode: diff --git a/scripts/stablevideodiffusion.py b/scripts/stablevideodiffusion.py index 16077cc2c..116bf9636 100644 --- a/scripts/stablevideodiffusion.py +++ b/scripts/stablevideodiffusion.py @@ -44,7 +44,8 @@ class Script(scripts.Script): return [num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] def run(self, p: processing.StableDiffusionProcessing, num_frames, override_resolution, min_guidance_scale, max_guidance_scale, decode_chunk_size, motion_bucket_id, noise_aug_strength, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument - if shared.sd_model is None or shared.sd_model.__class__.__name__ != 'StableVideoDiffusionPipeline': + c = shared.sd_model.__class__.__name__ if shared.sd_model is not None else '' + if c != 'StableVideoDiffusionPipeline' and c != 'TextToVideoSDPipeline': return None if hasattr(p, 'init_images') and len(p.init_images) > 0: if override_resolution: @@ -53,9 +54,13 @@ class Script(scripts.Script): p.task_args['image'] = images.resize_image(resize_mode=2, im=p.init_images[0], width=p.width, height=p.height, upscaler_name=None, output_type='pil') else: p.task_args['image'] = p.init_images[0] - p.ops.append('svd') + p.ops.append('stablevideo') p.do_not_save_grid = True - p.sampler_name = 'Default' # svd does not support non-default sampler + if c == 'StableVideoDiffusionPipeline': + p.sampler_name = 'Default' # svd does not support non-default sampler + p.task_args['output_type'] = 'np' + else: + p.task_args['output_type'] = 'pil' p.task_args['generator'] = torch.manual_seed(p.seed) # svd does not support gpu based generator p.task_args['width'] = p.width p.task_args['height'] = p.height @@ -66,7 +71,6 @@ class Script(scripts.Script): p.task_args['num_inference_steps'] = p.steps p.task_args['min_guidance_scale'] = min_guidance_scale p.task_args['max_guidance_scale'] = max_guidance_scale - p.task_args['output_type'] = 'np' shared.log.debug(f'StableVideo: args={p.task_args}') shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) processed = processing.process_images(p)