^(j\ d Z ddlZddlZddlZddlZddlmZ ddlZddl Z ddddddd Z
dd
Z i fd Z i fd Z
d
e j dee de j fdZddZe j& dfdZd Zy)a
This file is part of ComfyUI.
Copyright (C) 2024 Comfy
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
Nmlp_fc1mlp_fc2self_attn_k_projself_attn_q_projself_attn_v_projself_attn_out_proj)zmlp.fc1zmlp.fc2zself_attn.k_projzself_attn.q_projzself_attn.v_projzself_attn.out_projc
V i }t }|D ]V }dj | }d }|| j v r$| | j }|j | dj | }d } || j v r| | } |j | t
j D ]= }
|
j || || | } | | ||| < |j | j ? dj | } dj | }
| j | d }| j |
d }|V|j | d|ff||| < |8|j |
d|ff|dj || d t d < dj | }| j |d }| d|ff||| < |j | d j | }| j |d }|8d|ff|dj || d t d < |j | d
j | }| j |d }|;d |ff||| < |j | Y |r>| j D ]+ }||vst j d j | - |S )
Nz{}.alphaz
{}.dora_scalez {}.w_normz {}.b_normdiffz{}.bias.weightz{}.diffz {}.diff_bz
{}.set_weightsetzlora key not loaded: {})r formatkeysitemaddweight_adapteradaptersloadupdate loaded_keysgetlenloggingwarning)lorato_load log_missing
patch_dictr x
alpha_namealphadora_scale_name
dora_scale adapter_clsadapter w_norm_name b_norm_namew_normb_norm diff_name diff_weightdiff_bias_name diff_biasset_weight_name
set_weights 4/Users/danicosta/Desktop/Flux2/ComfyUI/comfy/lora.py load_lorar0 % s J%K
&&q)
$$))+E OOJ ')003
diik ) o.J OOO ,)22K!&&q$z;OG")0
71:&""7#6#67 3 "((+ !((+ +t,+t, OOK (&,vi%8Jwqz "! ,NTW]V_M`
9++GAJ7GY ,HIJ $$Q' hhy$/ "&,{n%=Jwqz " OOI &$++A.HH^T2 JPS\R^I_Jy''
3CS^O(DE F OON +)003XXot4
!&+j]%;Jwqz " OOO ,a d A # 9 @ @ CD c
6 | j j }t }|D ] }|j d s||dj |d t d < |j
d }|dkD r5|j d s$||dj ||dz t d < |j |j d d d}d }d }t d
D ] } t D ]| }
d j | |
}||v rS|j | t |
} ||| < d j | t |
} ||| < d
j | |
} ||| < dj | |
}||v rU|j | t |
} ||| < d j | t |
} ||| < d}d
j | |
} ||| < dj | |
}||v sd}|r7dj | t |
} ||| < dj | |
} ||| < *dj | t |
} ||| < d
j | |
} ||| < dj | t |
} ||| < |D ] }|j d s|j d r}|t d t d } d}
|r|
dz
}
|r4|
dz
}
|
dk( r*||dj |
| j dd < |
dz
}
||dj |
| j dd < |j d s|t d t d } dj | j dd } ||| < t | dk( rdj t t | }|D ]b }|j d s|j | s'|t | t d } ||dj | j dd < d d }||v r
||d < ||d < d }||v r||d < |S )!Nr ztext_encoders.{}z
.transformer.r clip_ .z'lora_te_text_model_encoder_layers_{}_{}F z9clip_h.transformer.text_model.encoder.layers.{}.{}.weightz(lora_te1_text_model_encoder_layers_{}_{}z,text_encoder.text_model.encoder.layers.{}.{}z9clip_l.transformer.text_model.encoder.layers.{}.{}.weightTz9clip_g.transformer.text_model.encoder.layers.{}.{}.weightz(lora_te2_text_model_encoder_layers_{}_{}z.text_encoder_2.text_model.encoder.layers.{}.{}z-lora_prior_te_text_model_encoder_layers_{}_{}zt5xxl.transformer. z lora_te{}_{}_z hydit_clip.transformer.bert.z lora_te1_{}z{}.transformer.model.z
lora_te_{}z)clip_g.transformer.text_projection.weight lora_prior_te_text_projectionlora_te2_text_projectionz)clip_l.transformer.text_projection.weightlora_te1_text_projection)
state_dictr r endswithr
r find
startswithr splitrange
LORA_CLIP_MAPreplacenextiter)modelkey_mapsdk
prefix_setktptext_model_lora_keyclip_l_presentclip_g_presentbclora_keyl_keyt5_index full_prefixs r/ model_lora_keys_cliprU a sl
!
!
#CJ
::i FGG&--a0@#i. .AB C(BAvall7 3PQ*11!BFC N?2KLM NN1773<? +
DNN
2Y AKRRSTVWXACx.55a q9IJ$%!ELLQP]^_P`a$%!IPPQRTUV$%!KRRSTVWXACx.55a q9IJ$%!ELLQP]^_P`a$%!!%IPPQRTUV$%!KRRSTVWXACx!%! I P PQRTabcTd eH()GH% O V VWXZ[ \H()GH% H O OPQS`abSc dH()GH% M T TUVXY ZH()GH% N U UVWYfghYi jH()GH%E J ::i || 01# 23S^OD ! MH! MH 1}\] 5 5h
cSV@W XY A
TU--h
c38OPQ <=# <=s9~oN(//
c30GH$%!# & :!-44T$z:J5KL Azz)$<< , c+ .I ?ELMGL / /
c30G HI 4ACx34/0./*+3ACx./*+ Nr1 c
* | j }|j }|D ] }|j d s|j d r^|t d t d j dd }||dj
| < ||dj
|d t d < ||dj
| < t j j | j j }|D ] }|j d sdj
|| }|d t d j dd }||dj
| < ||dj
| < d d
g}|D ]L } d j
| |d t d j d d
}
|
j d r|
d d }
|||
< N t | t j j rg|D ]b }|j d s|j d s'|t d t d j dd }||dj
| < d t | t j j rt j j! | j j d }|D ] }|j d s|| } dj
|d t d }| ||< dj
|d t d }| ||< dj
|d t d j dd }| ||< dj
|d t d j dd }| ||< t | t j j" rvt j j% | j j d }|D ]< }|j d s|| } dj
|d t d }| ||< > t | t j j& rt j j) | j j d }|D ] }|j d s|| } dj
|d t d }| ||< dj
|d t d }| ||< dj
|d t d }| ||< t | t j j* rW|D ]R }|j d s|j d s'|t d t d }||dj
| < T t | t j j, rXt j j/ | j j d }|D ] }|j d s|| } | |dj
|d t d < | |dj
|d t d j dd < | |dj
|d t d j dd < | ||d t d < |D ]k }| j j j1 dd } |j d s;d|v s@|dd| dz ff|dj
|j dd < m t | t j j2 rW|D ]R }|j d s|j d s'|t d t d }||dj
| < T t | t j j4 r|D ] }|j d s|j d s'|}|j d d j d d j d d! }|j d"d# j d$d% }|j d&d' }|t d t d }||dj
| < ||dj
| < t | t j j6 r{|D ]v }|j d s|j d s'|t d t d }||dj
|j dd < ||dj
| < x t | t j j8 rW|D ]R }|j d s|j d s'|t d t d }||dj
| < T t | t j j: rW|D ]R }|j d s|j d s'|t d t d }||dj
| < T t | t j j< r|D ] }|j d s|j d s'|t d t d }||dj
| < ||dj
| < ||dj
|j dd < t | t j j> rt j jA | j j d }|D ]y }|j d s|| } |d t d }| |dj
| < | |dj
| < | |dj
|j dd < | ||< { t | t j jB rk|D ]f }|j d s|j d s'|t d t d }||dj
| < ||dj
| < h t | t j jD r{|D ]v }|j d( s|j d s'|t d( t d }||dj
| < ||dj
|j dd < x t | t j jF rW|D ]R }|j d s|j d s'|t d t d }||dj
| < T |S ))Nzdiffusion_model.r r5 r8 z lora_unet_{}z{}zdiffusion_model.{}z
lycoris_{} zunet.z{}{}z.to_z.processor.to_z .to_out.0zlora_prior_unet_{})
output_prefixztransformer.{}zbase_model.model.{}zlora_transformer_{}zunet.base_model.model.{} hidden_sizer z .linear1. z.linear1.weightz .linear1_qkvz _mod.lin.z _mod.linear.z
_attn.qkv.z
_attn_qkv.z _attn.proj.z _attn_proj.zmlp.0.zmlp.fc1.zmlp.2.zmlp.fc2.z.modulation.lin.z.modulation.linear.zdiffusion_model.decoder.)$r< r r? r= r rC r
comfyutilsunet_to_diffusers model_config unet_config
isinstance
model_baseStableCascade_CSD3mmdit_to_diffusersAuraFlowauraflow_to_diffusersPixArtpixart_to_diffusers
HunyuanDiTFluxflux_to_diffusersr
GenmoMochi HunyuanVideoHiDreamACEStepOmnigen2 QwenImageLumina2z_image_to_diffusers
Kandinsky5 ACEStep15
ErnieImage)
rF rG sdrH rJ key_loradiffusers_keysunet_keydiffusers_lora_prefixpdiffusers_lora_keytorZ s
r/ model_lora_keys_unetr s B
'')C
<<* +zz)$ S!3 4c)n_EMMcSVW;<--h78<= A&6I$789*+ A' [[2253E3E3Q3QRN
::i +22>!3DEH)3y>/*223//B/J/J3PS/TU$&! %))223 ::5;M;M;Y;Yi{:| Azz)$#A&+2215Es9~o3FG$&! %))001 889K9K9W9Wgy8z Azz)$#A&+2215Es9~o3FG$&!077:JC N?8KL$&!5<?
%))../ 66u7I7I7U7Uew6x Azz)$#A&HJ(//2BC N?0CDEVX ++A.>I,?,G,GS,QRS_a-44Q7GY 5H5P5PQTVY5Z[\/1*C N?+,
A ,,88<<]ANKzz)$)9VWZ[]^`kno`oYpUq AII.? $PQR
%))445A|| ./AJJy4I S!3 4c)n_E12 H-.
%))667A|| ./AJJy4I #++K HPPQ]_kltt vC ER S#++HjAII(T^_#++,>@UV#C(:$;S^OL=>(//9:AB,33H=> %))112A|| ./::i( %7!8#i. IHOPGL / /0@0@c0J KLABG , 3 3H => %))112A|| ./AJJy4I S!3 4c)n_E12 H-.
%))223A|| ./AJJy4I S!3 4c)n_E12 H-.
%))334A|| ./AJJy4I S!3 4c)n_E12 H-.=>(//9:KL ++H,<,=?(//9:KM ++H,<,(//9: %))334A|| 67AJJy?KL ++H,<,(//9:
Nr1 tensor new_shapereturnc t t t | D cg c] }|| | j | k c} r t d t | t | j k7 r t d t j || j | j }t d | j D }t d | j D }| | ||< |S c c}w )a
Pad a tensor to a new shape with zeros.
Args:
tensor (torch.Tensor): The original tensor to be padded.
new_shape (List[int]): The desired shape of the padded tensor.
Returns:
torch.Tensor: A new tensor padded with zeros to the specified shape.
Note:
If the new shape is smaller than the original tensor in any dimension,
the original tensor will be truncated in that dimension.
zGThe new shape must be larger than the original tensor in all dimensionszLThe new shape must have the same number of dimensions as the original tensor)dtypedevicec 3 4 K | ] }t d | ywr Nslice.0dims r/ z&pad_tensor_to_shape.. s > #a
c 3 4 K | ] }t d | ywr r r s r/ r z&pad_tensor_to_shape.. s = uQ} r )
anyrA r shape
ValueErrortorchzerosr r tuple)r r i
padded_tensor orig_slices
new_slicess r/ pad_tensor_to_shaper k s E#i.4I J4IqIaL6<<? *4I JKbcc
9~V\\**ghh KK fmmTM > >>K= ==J !'{ 3M* ! Ks C$c R |j }| D ] }|d }|d }|t |t r t |t j r|j | }||}Pt
| dk( s_|d |d }
} | dk( sot
|
dkD s~|
d d s|
d j } |S )Nr4 r[ r7 r r
pad_weight)r ra listr WeightAdapterBasecalculate_shaper ) patchesweightkeyoriginal_weights
current_shaper} voffset
adapter_shape
patch_type
patch_datas r/ r r s LLM
aD1 At!4 a 99 : - -c 2M ( -
q6Q;%&qT1Q4
JV#J!(;
1
l@[ *1
3 3
% ( r1 c | D ]` }|d }|d }|d }|d } |d }
|
d }
d } | |} |j | d | d | d }|dk7 r||z }t |t rRt |dd |d d t j
j
|d d |j |d d
|| f}t |t j rR|j ||||| |
|| } | 0t j d j |j | n| }| | } t | dk( rd
}
nt | dk( r
|d }
|d }
d
k( r%|d }t | dkD xr |d d }|ri|j |j k7 rPt j dj ||j |j t# ||j }|dk7 r|j |j k7 rz"calculate_weight.. s r1 g ?T)copy)inplace)intermediate_dtypez Calculate Weight Failed: {} {}r
r z"Pad weight {} from {} to shape: {}g z4WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}r
model_as_loraz patch type not recognized {} {})narrowra r calculate_weightr\ model_managementcast_to_devicer r r r r r
namer r infor r copy_)r r r r r r} strengthr strength_modelr function
old_weightoutputr r
do_pad_weight
target_weightr* s r/ r r s
Q4
aD11Q4 "H
J]]6!9fQiCF S n $F a !!AB%1a1G1G1V1VWXYZW[\]W^`f`m`m pB IM1V 2N X\ *] _b wI J MA a 99 :''X~vW_as vF GF~ @ G GPS TU ) 'F q6Q; J
Vq[1J!A !"1D FQJ =1Q4 +=M v||!; A H Hfll\`\f\f gh,VTZZ@3:: -OO$Z$a$abegkgqgqsyss %A Bhx%2H2H2W2WX\^d^k^kmsmymy2z'z {{F
5
LL1
?
**+A$M 00??
v}}^pq 00??@PQT@UVW@XYZ@[]c]j]jl~@K hx%*@*@*O*OP[]c]j]jlrlxlx*y yz zF OO = D DZQT U V ! Fy | Mr1 c
t | t j rt j j | }d }dxx |z
cc< | S |||z }r$t j j | g|d t j j | g| d S t | t j r3 t | | j t | j S t | t rt fd| D S t | t r | D cg c] }t | c}S | S c c}w )Nr T) non_blockingstreamc 3 < K | ] }t | y wr )prefetch_prepared_value)r r r counter destinationr s r/ r z*prefetch_prepared_value.. s$ ich[_,T7KQUVchs )ra r Tensorr\ memory_managementvram_aligned_sizer cast_to_gatheredinterpret_gathered_liker r typer r weightsr r ) valuer r r r sizer destr s ```` r/ r r s3 % &&&88?
d
L6&4-0 " " 3 3UGTPT]c 3 d&&>>wMaPP E>;; <tE{5,,.EemmU\^ikqsw.xyy E5 !ichiii E4 ^cd^cVZ'g{FDQ^cdd L es .E )Tr )__doc__comfy.memory_managementr\ comfy.utilscomfy.model_managementcomfy.model_basecomfy.weight_adapterr r r rB r0 rU r r r intr r float32r r r r1 r/ r s $ - ***.
:x )+ Xt )+ m` c u|| B2 ?Dmm^b ?Br1