
    ^(j]             	       
   d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlZd dlm	Z	 ddl
mZ ddl
mZ dd	l
mZ d dlZd dlZd dlZd d
lmZ d ZdWdZdXdZdYdZdZdZd[dZd Zd\dZd]dZ G d d      Z G d d      Zd Z d Z!d^dZ"dejF                  dejF                  fdZ$dejF                  dejF                  fd Z% ejL                         ddddd e'd!      dfd"       Z( ejL                         d_d#       Z) ejL                         d_d$       Z* ejL                         ddddd e'd!      dfd%       Z+ ejL                         ddddd e'd!      dfd&       Z, ejL                         d_d'       Z- ejL                         d_d(       Z.d) Z/ ejL                         d`d*       Z0 G d+ d,      Z1 G d- d.ejd                        Z3 ejL                         dad/       Z4 ejL                         dbd0       Z5 ejL                         d_d1       Z6 ejL                         d_d2       Z7 ejL                         dcd3       Z8 ejL                         ddd4       Z9 ejL                         ded5       Z: ejL                         dfd6       Z; ejL                         d_d7       Z< ejL                         d_d8       Z= ejL                         dfd9       Z> ejL                         ded:       Z? ejL                         dcd;       Z@d< ZAdgd=ZB ejL                         dhd>       ZC ejL                         did?       ZD ejL                         ddddd e'd!      dfd@       ZEd`dAZFd`dBZG ejL                         djdC       ZH ejL                         d_dD       ZI ejL                         dddE       ZJ ejL                         d_dF       ZK ejL                         dddG       ZL ejL                         dkdH       ZM ejL                         dldI       ZN ejL                         dldJ       ZO ejL                         d_dK       ZP ejL                         d_dL       ZQ ejL                         dmdM       ZR ejL                         dndN       ZS ejL                         dodO       ZT ejL                         dpdP       ZU ejL                         dqdQ       ZV ejL                         drdR       ZW ejL                         dsdS       ZX ejL                         dtdT       ZY ejL                         dudU       ZZ ejL                         	 	 dvdV       Z[y)w    N)partial)	integrate)nn)tqdm   )utils)deis)	sa_solver)model_trangec                 P    t        j                  | | j                  dg      g      S Nr   )torchcat	new_zerosxs    D/Users/danicosta/Desktop/Flux2/ComfyUI/comfy/k_diffusion/sampling.pyappend_zeror      s!    99aaS)*++    c                     t        j                  dd| |      }|d|z  z  }|d|z  z  }||||z
  z  z   |z  }t        |      j                  |      S )z6Constructs the noise schedule of Karras et al. (2022).r   r   device)r   linspacer   to)	n	sigma_min	sigma_maxrhor   rampmin_inv_rhomax_inv_rhosigmass	            r   get_sigmas_karrasr#      sa    >>!Q&1DC(KC(KDK+$=>>3FFv!!&))r   c                     t        j                  t        j                  |      t        j                  |      | |      j	                         }t        |      S )z)Constructs an exponential noise schedule.r   )r   r   mathlogexpr   )r   r   r   r   r"   s        r   get_sigmas_exponentialr(       s=    ^^DHHY/)1DaPVW[[]Fvr         ?c                     t        j                  dd| |      |z  }t        j                  |t        j                  |      t        j                  |      z
  z  t        j                  |      z         }t        |      S )z5Constructs an polynomial in log sigma noise schedule.r   r   r   )r   r   r'   r%   r&   r   )r   r   r   r   r   r   r"   s          r   get_sigmas_polyexponentialr+   &   s_    >>!Q&1S8DYYttxx	2TXXi5HHIDHHU^L__`Fvr   c                     t        j                  d|| |      }t        j                  t         j                  j	                  ||dz  z  dz  ||z  z               }t        |      S )z*Constructs a continuous VP noise schedule.r   r      )r   r   sqrtspecialexpm1r   )r   beta_dbeta_mineps_sr   tr"   s          r   get_sigmas_vpr5   -   sT    q%62AZZ++FQ!VOa,?(Q,,NOPFvr           c           
      (   d}t        j                  dd| |      }fd}||t        j                  d|z
        z  t        j                  ddt        j                  d|z
        z  z
  |z         z  z
  }	 |t        j
                  |	            }
|
S )zAConstructs the noise schedule proposed by Tiankai et al. (2024). h㈵>r   r   r   c                 4    t        j                  |       S )Nminmax)r   clamp)r   r   r   s    r   <lambda>z$get_sigmas_laplace.<locals>.<lambda>8   s    ekk!	Br         ?r-   )r   r   signr&   absr'   )r   r   r   mubetar   epsilonr   r=   lmbr"   s    ``        r   get_sigmas_laplacerF   4   s~    Gq!Qv.ABE
tejjQ''%))AEIIc!e<L8L4Lw4V*WW
WC599S>"FMr   c                 N    | |z
  t        j                  || j                        z  S )z6Converts a denoiser output to a Karras ODE derivative.)r   append_dimsndim)r   sigmadenoiseds      r   to_drL   ?   s"    LE--eQVV<<<r   c                     |s|dfS t        |||dz  | dz  |dz  z
  z  | dz  z  dz  z        }|dz  |dz  z
  dz  }||fS )zCalculates the noise level (sigma_down) to step down to and the amount
    of noise to add (sigma_up) when doing an ancestral sampling step.r6   r-   r?   )r;   )
sigma_fromsigma_toetasigma_up
sigma_downs        r   get_ancestral_steprS   D   sm     |8SHMZ1_xST}5T$UXbfgXg$glo#oopHa-(a-/C7Jxr   c                      |Y j                   t        j                   d      k(  r|dz  }t        j                   j                         j                  |       nd  fdS )Ncpur   r   c                     t        j                  j                         j                  j                  j
                        S )N)dtypelayoutr   	generator)r   randnsizerW   rX   r   )rJ   
sigma_nextrY   r   s     r   r>   z'default_noise_sampler.<locals>.<lambda>X   s2    U[[QRQYQYbcbjbjv  &Ar   )r   r   	Generatormanual_seed)r   seedrY   s   ` @r   default_noise_samplerr`   N   sX    88u||E**AIDOO1884	d#	 A  Ar   c                   .    e Zd ZdZddZed        Zd Zy)BatchedBrownianTreezGA wrapper around torchsde.BrownianTree that enables batches of entropy.Nc                    j                  dd      | _        | j                        \  | _        j                  dd       t	        j
                  |      d| _        |'t	        j                  ddd      j                         f}nLt        |t        t        f      r3t        |      |j                  d   k7  rt        d      d| _        d   n|f}| j                  r[j                         j!                         j                         j!                         j                         j!                         ct        fd	|D              | _        y )
NrU   Tw0Fr   l     zbPassing a list or tuple of seeds to BatchedBrownianTree requires a length matching the batch size.c              3   R   K   | ]  }t        j                  fd |i   yw)entropyN)torchsdeBrownianTree).0skwargst0t1rd   s     r   	<genexpr>z/BatchedBrownianTree.__init__.<locals>.<genexpr>p   s.     `[_VW800RQQQ&Q[_s   $')popcpu_treesortr@   r   
zeros_likebatchedrandintitem
isinstancetuplelistlenshape
ValueErrordetachrU   trees)selfr   rm   rn   r_   rl   rd   s     `` `@r   __init__zBatchedBrownianTree.__init__^   s   

5$/ IIb"-B	ZZd#:!!!$B<MM!["5::<>Dudm,4yAGGAJ&   "F  G  GDLAB7D==*BIIKOO,=ryy{?PJBB`[_``
r   c                      | |k  r| |dfS || dfS Nr   re   )abs     r   rr   zBatchedBrownianTree.sortr   s    E1ay11bz1r   c           	          | j                  ||      \  }}}|j                  |j                  }}| j                  rX|j	                         j                         j                         |j	                         j                         j                         }}t        j                  | j                  D cg c]  } |||       c}      j                  ||      | j                  |z  z  }| j                  r|S |d   S c c}w )Nr   rW   r   )rr   r   rW   rq   r}   rU   floatr   stackr~   r   r@   rt   )r   rm   rn   r@   r   rW   treews           r   __call__zBatchedBrownianTree.__call__v   s    yyR(B		288==YY[__&,,.		0A0G0G0IBKK$**=*$b"*=>AAW\A]aeajajmqaqrLLq*ad* >s   0C;N)__name__
__module____qualname____doc__r   staticmethodrr   r   re   r   r   rb   rb   [   s$    Qa( 2 2+r   rb   c                   &    e Zd ZdZdd dfdZd Zy)BrownianTreeNoiseSampleras  A noise sampler backed by a torchsde.BrownianTree.

    Args:
        x (Tensor): The tensor whose shape, device and dtype to use to generate
            random samples.
        sigma_min (float): The low end of the valid interval.
        sigma_max (float): The high end of the valid interval.
        seed (int or List[int]): The random seed. If a list of seeds is
            supplied instead of a single integer, then the noise sampler will
            use one BrownianTree per batch item, each with its own seed.
        transform (callable): A function that maps sigma to the sampler's
            internal timestep.
    Nc                     | S r   re   r   s    r   r>   z!BrownianTreeNoiseSampler.<lambda>   s    qr   Fc                     || _         | j                  t        j                  |            | j                  t        j                  |            }}t        |||||      | _        y )NrU   )	transformr   	as_tensorrb   r   )	r   r   r   r   r_   r   rU   rm   rn   s	            r   r   z!BrownianTreeNoiseSampler.__init__   sK    "	 :;T^^EOO\eLf=gB'2r4SA	r   c                     | j                  t        j                  |            | j                  t        j                  |            }}| j                  ||      ||z
  j	                         j                         z  S r   )r   r   r   r   rA   r.   )r   rJ   r\   rm   rn   s        r   r   z!BrownianTreeNoiseSampler.__call__   sY     67XbHc9dByyR BG==?#7#7#999r   )r   r   r   r   r   r   re   r   r   r   r      s     6:[V[ B
:r   r   c                     t        |t        j                  j                        r| j	                         j                         S | j                         j                         S )z4Convert sigma to half-logSNR log(alpha_t / sigma_t).)rw   comfymodel_samplingCONSTlogitnegr&   )rJ   r   s     r   sigma_to_half_log_snrr      s@    .%"6"6"<"<={{}  ""99;??r   c                     t        |t        j                  j                        r| j	                         j                         S | j	                         j                         S )z4Convert half-logSNR log(alpha_t / sigma_t) to sigma.)rw   r   r   r   r   sigmoidr'   )half_log_snrr   s     r   half_log_snr_to_sigmar      sH    .%"6"6"<"<=!))++!!##r   c                     t        |       dk  r| S t        |t        j                  j                        r,| d   dk\  r$| j                         } |j                  |      | d<   | S )z/Adjust the first sigma to avoid invalid logSNR.r   r   )rz   rw   r   r   r   clonepercent_to_sigma)r"   r   percent_offsets      r   offset_first_sigma_for_snrr      sX    
6{a.%"6"6"<"<=!9>\\^F&77GF1IMr   hreturnc                 ,    t        j                  |       S )zCCompute the result of h*phi_1(h) in exponential integrator methods.r   r0   r   s    r   
ei_h_phi_1r      s    ;;q>r   c                 8    t        j                  |       | z
  | z  S )zCCompute the result of h*phi_2(h) in exponential integrator methods.r   r   s    r   
ei_h_phi_2r      s    KKNQ!##r   infc
           	          |i n|}|j                  |j                  d   g      }
t        t        |      dz
  |      D ]  }|dkD  r:|||   cxk  r|k  rn nt	        |t        |      dz
  z  d      nd}||   |dz   z  }nd}||   }|dkD  r/t        j                  |      |	z  }|||dz  ||   dz  z
  dz  z  z   } | |||
z  fi |}t        |||      }| |||||   ||d       ||dz      |z
  }|||z  z   } |S )	z?Implements Algorithm 2 (Euler steps) from Karras et al. (2022).r   r   disable4y?r6   r-   r?   r   irJ   	sigma_hatrK   new_onesr{   trangerz   r;   r   
randn_likerL   )modelr   r"   
extra_argscallbackr   s_churns_tmins_tmaxs_noises_inr   gammar   epsrK   ddts                     r   sample_eulerr      sA    ")zJ::qwwqzl#DCK!OW5Q;FLPVWXPYFc]cFcC3v;?3\BikEq	UQY/IEq	I19""1%/CC9>F1IN:sBBBAI,;
;Ix(11vayy^fghAE]Y&BJ# 6$ Hr   c	                    t        | j                  j                  j                  t        j                  j                        rt        | ||||||||	      S 	 |i n|}|j                  dd       }	|t        ||	      n|}|j                  |j                  d   g      }
t        t        |      dz
  |      D ]  } | |||   |
z  fi |}t        ||   ||dz      |      \  }}| |||||   ||   |d       |dk(  r|}Kt        |||   |      }|||   z
  }|||z  z    |||   ||dz            |z  |z  z   } |S )Nr_   r_   r   r   r   rP   r   )rw   inner_modelr   r   r   sample_euler_ancestral_RFgetr`   r   r{   r   rz   rS   rL   )r   r   r"   r   r   r   rP   r   noise_samplerr_   r   r   rK   rR   rQ   r   r   s                    r   sample_euler_ancestralr      sm   %##//>>@T@T@Z@Z[(6:xQXZ]_fhuvv5!)zJ>>&$'D;H;P)!$7VcM::qwwqzl#DCK!OW5F1I,;
;1&)VAE]PST
H11vayvay^fgh?AQq	8,AfQi'BAF
]6!9fQUmDwNQYYYA 6 Hr   c	           	         |i n|}|j                  dd      }	|t        ||	      n|}|t        | j                  j                  j                  d      dd      z  }|j                  |j                  d   g      }
t        t        |      dz
  |	      D ]  } | |||   |
z  fi |}| |||||   ||   |d
       ||dz      dk(  r|}7d||dz      ||   z  dz
  |z  z   }||dz      |z  }d||dz      z
  }d|z
  }||dz      dz  |dz  |dz  z  |dz  z  z
  dz  }|||   z  }||z  d|z
  |z  z   }|dkD  s||z  |z   |||   ||dz            |z  |z  z   } |S )z+Ancestral sampling with Euler method steps.Nr_   r   r   noise_scaler)   r   r   r   r   r-   r?   )
r   r`   getattrr   model_patcherget_model_objectr   r{   r   rz   )r   r   r"   r   r   r   rP   r   r   r_   r   r   rK   downstep_ratiorR   	alpha_ip1
alpha_downrenoise_coeffsigma_down_i_ratios                      r   r   r      s    ")zJ>>&$'D;H;P)!$7VcM 1 1 ? ? P PQa bdqsvwwG::qwwqzl#DCK!OW5F1I,;
;11vayvay^fgh!a%=AA&Q-&)";a"?3!FFNA7JF1q5M)IZJ#AE]A-
A	10Lz[\}0\\_bbM!+fQi!7"Q&!.@*@H)LLAQw+q0=FSTWXSXM3Z]d3dgt3tt% 6& Hr   c
           	         |i n|}|j                  |j                  d   g      }
t        t        |      dz
  |      D ]  }|dkD  r:|||   cxk  r|k  rn nt	        |t        |      dz
  z  d      nd}||   |dz   z  }nd}||   }||   |dz   z  }|dkD  r/t        j                  |      |	z  }|||dz  ||   dz  z
  dz  z  z   } | |||
z  fi |}t        |||      }| |||||   ||d       ||dz      |z
  }||dz      dk(  r	|||z  z   }|||z  z   } | |||dz      |
z  fi |}t        |||dz      |      }||z   dz  }|||z  z   } |S )	z>Implements Algorithm 2 (Heun steps) from Karras et al. (2022).r   r   r   r   r6   r-   r?   r   r   )r   r   r"   r   r   r   r   r   r   r   r   r   r   r   r   rK   r   r   x_2
denoised_2d_2d_primes                         r   sample_heunr     s    ")zJ::qwwqzl#DCK!OW55Q;FLPVWXPYFc]cFcC3v;?3\BikEq	UQY/IEq	I1I+	19""1%/CC9>F1IN:sBBBAI,;
;Ix(11vayy^fghAE]Y&!a%=AAF
A a"f*CsF1q5MD$8GJGJsF1q5M:6C3w!mGGbL A5 66 Hr   c
           	         |i n|}|j                  |j                  d   g      }
t        t        |      dz
  |      D ]E  }|dkD  r:|||   cxk  r|k  rn nt	        |t        |      dz
  z  d      nd}||   |dz   z  }nd}||   }|dkD  r/t        j                  |      |	z  }|||dz  ||   dz  z
  dz  z  z   } | |||
z  fi |}t        |||      }| |||||   ||d       ||dz      dk(  r||dz      |z
  }|||z  z   }|j                         j                  ||dz      j                         d      j                         }||z
  }||dz      |z
  }|||z  z   } | |||
z  fi |}t        |||      }|||z  z   }H |S )	zMA sampler inspired by DPM-Solver-2 and Algorithm 2 from Karras et al. (2022).r   r   r   r   r6   r-   r?   r   )r   r{   r   rz   r;   r   r   rL   r&   lerpr'   )r   r   r"   r   r   r   r   r   r   r   r   r   r   r   r   rK   r   r   	sigma_middt_1dt_2r   r   r   s                           r   sample_dpm_2r   /  s    ")zJ::qwwqzl#DCK!OW55Q;FLPVWXPYFc]cFcC3v;?3\BikEq	UQY/IEq	I19""1%/CC9>F1IN:sBBBAI,;
;Ix(11vayy^fgh!a%=AA*BAF
A ",,VAE]->->-@#FJJLIy(D!a%=9,Da$h,CsI$4C
CJsIz2CC$JA7 68 Hr   c	                 r   t        | j                  j                  j                  t        j                  j                        rt        | ||||||||	      S 	 |i n|}|j                  dd       }	|t        ||	      n|}|j                  |j                  d   g      }
t        t        |      dz
  |      D ]  } | |||   |
z  fi |}t        ||   ||dz      |      \  }}| |||||   ||   |d       t        |||   |      }|dk(  r|||   z
  }|||z  z   }i||   j                         j                  |j                         d      j!                         }|||   z
  }|||   z
  }|||z  z   } | |||
z  fi |}t        |||      }|||z  z   }| |||   ||dz            |z  |z  z   } |S )	Nr_   r   r   r   r   r   r   r?   )rw   r   r   r   r   sample_dpm_2_ancestral_RFr   r`   r   r{   r   rz   rS   rL   r&   r   r'   )r   r   r"   r   r   r   rP   r   r   r_   r   r   rK   rR   rQ   r   r   r   r   r   r   r   r   s                          r   sample_dpm_2_ancestralr   S  s   %##//>>@T@T@Z@Z[(6:xQXZ]_fhuvv@!)zJ>>&$'D;H;P)!$7VcM::qwwqzl#DCK!OW5F1I,;
;1&)VAE]PST
H11vayvay^fghF1Ix(?fQi'BAF
A q	,,Z^^-=sCGGIIvay(Dq	)Da$h,CsI$4C
CJsIz2CC$JAM&)VAE];gEPPA' 6( Hr   c	           	         |i n|}|j                  dd      }	|t        ||	      n|}|t        | j                  j                  j                  d      dd      z  }|j                  |j                  d   g      }
t        t        |      dz
  |	      D ];  } | |||   |
z  fi |}d||dz      ||   z  dz
  |z  z   }||dz      |z  }d||dz      z
  }d|z
  }||dz      d
z  |d
z  |d
z  z  |d
z  z  z
  dz  }| |||||   ||   |d       t        |||   |      }|dk(  r|||   z
  }|||z  z   }||   j                         j                  |j                         d      j                         }|||   z
  }|||   z
  }|||z  z   } | |||
z  fi |}t        |||      }|||z  z   }||z  |z   |||   ||dz            |z  |z  z   }> |S )z6Ancestral sampling with DPM-Solver second-order steps.Nr_   r   r   r   r)   r   r   r   r-   r?   r   )r   r`   r   r   r   r   r   r{   r   rz   rL   r&   r   r'   )r   r   r"   r   r   r   rP   r   r   r_   r   r   rK   r   rR   r   r   r   r   r   r   r   r   r   r   r   s                             r   r   r   s  sM    ")zJ>>&$'D;H;P)!$7VcM 1 1 ? ? P PQa bdqsvwwG::qwwqzl#DCK!OW55F1I,;
;fQqSk&)3a73>>AaC[>1
qsO	^
!a*a-	1*DZQR]*RRUXX11vayvay^fghF1Ix(?fQi'BAF
A q	,,Z^^-=sCGGIIvay(Dq	)Da$h,CsI$4C
CJsIz2CC$JA:%*]6!9fQQRUm-TW^-^an-nnA1 62 Hr   c                       dz
  kD  rt        d  d        fd}t        j                  |   dz      d      d   S )Nr   zOrder z too high for step c                 x    d}t              D ](  }|k(  r	|| |z
     z
  z
     |z
     z
  z  z  }* |S Nr)   )range)tauprodkr   jorderr4   s      r   fnz"linear_multistep_coeff.<locals>.fn  sW    uAAvS1QU8^!a%1QU8(;<<D  r   -C6?)epsrelr   )r|   r   quad)r   r4   r   r   r   s   ```` r   linear_multistep_coeffr     sT    qy1}6%(;A3?@@ >>"adAa!eHT:1==r   c                    |i n|}|j                  |j                  d   g      }|j                         j                         j	                         }g }	t        t        |      dz
  |      D ]  }
 | |||
   |z  fi |}t        |||
   |      }|	j                  |       t        |	      |kD  r|	j                  d       | |||
||
   ||
   |d       ||
dz      dk(  r|}wt        |
dz   |      }t        |      D cg c]  }t        |||
|       }}|t        d t        |t        |	            D              z   } |S c c}w )Nr   r   r   r   c              3   ,   K   | ]  \  }}||z    y wr   re   )rj   coeffr   s      r   ro   zsample_lms.<locals>.<genexpr>  s     L2KheQ	2Ks   )r   r{   r}   rU   numpyr   rz   rL   appendrp   r;   r   r   sumzipreversed)r   r   r"   r   r   r   r   r   
sigmas_cpudsr   rK   r   	cur_orderr   coeffss                   r   
sample_lmsr    sJ   !)zJ::qwwqzl#D$$&,,.J	BCK!OW5F1I,;
;F1Ix(
		!r7U?FF1I11vayvay^fgh!a%=AAAE5)ISXYbScdSca,Y
AqIScFdCL#fhrl2KLLLA 6 H es    Ec                   $    e Zd ZdZddZd Zd Zy)PIDStepSizeControllerz4A PID controller for ODE adaptive step size control.c                     || _         ||z   |z   |z  | _        |d|z  z    |z  | _        ||z  | _        || _        || _        g | _        y )Nr-   )r   b1b2b3accept_safetyr   errs)r   r   pcoefficoeffdcoeffr   r  r   s           r   r   zPIDStepSizeController.__init__  sW    F?V+u4QZ'(505.*	r   c                 8    dt        j                  |dz
        z   S r   )r%   atan)r   r   s     r   limiterzPIDStepSizeController.limiter  s    499QU###r   c                    dt        |      | j                  z   z  }| j                  s
|||g| _        || j                  d<   | j                  d   | j                  z  | j                  d   | j                  z  z  | j                  d   | j
                  z  z  }| j                  |      }|| j                  k\  }|r8| j                  d   | j                  d<   | j                  d   | j                  d<   | xj                  |z  c_        |S )Nr   r   r-   )	r   r   r  r  r  r  r  r  r   )r   error	inv_errorfactoraccepts        r   propose_stepz"PIDStepSizeController.propose_step  s    u01	yy"Iy9DI 		!1(499Q<477+BBTYYq\UYU\U\E\\f%4---99Q<DIIaL99Q<DIIaL&r   N)r   Q?g:0yE>)r   r   r   r   r   r  r  re   r   r   r
  r
    s    >$r   r
  c                   ^     e Zd ZdZd fd	Zd Zd Zd ZddZddZ	ddZ
dd	Zdd
Z xZS )	DPMSolverz1DPM-Solver. See https://arxiv.org/abs/2206.00927.c                 b    t         |           || _        |i n|| _        || _        || _        y r   )superr   r   r   eps_callbackinfo_callback)r   r   r   r"  r#  	__class__s        r   r   zDPMSolver.__init__  s3    
 * 2"
(*r   c                 $    |j                          S r   r&   )r   rJ   s     r   r4   zDPMSolver.t  s    		|r   c                 >    |j                         j                         S r   r   r'   )r   r4   s     r   rJ   zDPMSolver.sigma  s    uuw{{}r   c                 2   ||v r||   |fS | j                  |      |j                  |j                  d   g      z  }| | j                  ||g|i | j                  |z
  | j                  |      z  }| j
                  | j                          |||i|fS )Nr   )rJ   r   r{   r   r   r"  )	r   	eps_cachekeyr   r4   argsrl   rJ   r   s	            r   r   zDPMSolver.eps  s    )S>9,,

1

AGGAJ< 88:4::aKKKFKKtzzZ[}\(S#++++r   c                     |i n|}||z
  }| j                  |d||      \  }}|| j                  |      |j                         z  |z  z
  }||fS )Nr   r   rJ   r0   )r   r   r4   t_nextr*  r   r   x_1s           r   dpm_solver_1_stepzDPMSolver.dpm_solver_1_step  s\    #+B	QJ)UAq9Y$**V$qwwy0366I~r   c                    |i n|}||z
  }| j                  |d||      \  }}|||z  z   }|| j                  |      ||z  j                         z  |z  z
  }	| j                  |d|	|      \  }
}|| j                  |      |j                         z  |z  z
  | j                  |      d|z  z  |j                         z  |
|z
  z  z
  }||fS )Nr   eps_r1r-   r.  )r   r   r4   r/  r1r*  r   r   s1u1r3  r   s               r   dpm_solver_2_stepzDPMSolver.dpm_solver_2_step  s    #+B	QJ)UAq9YaZB26.."22S88 HHY"bA	$**V$qwwy0366F9KqSUv9VYZY`Y`Yb9bflorfr9ssI~r   c                    |i n|}||z
  }| j                  |d||      \  }}|||z  z   }	|||z  z   }
|| j                  |	      ||z  j                         z  |z  z
  }| j                  |d||	      \  }}|| j                  |
      ||z  j                         z  |z  z
  | j                  |
      ||z  z  ||z  j                         ||z  z  dz
  z  ||z
  z  z
  }| j                  |d||
      \  }}|| j                  |      |j                         z  |z  z
  | j                  |      |z  |j                         |z  dz
  z  ||z
  z  z
  }||fS )Nr   r3  r   eps_r2r.  )r   r   r4   r/  r4  r2r*  r   r   r5  s2r6  r3  u2r9  x_3s                   r   dpm_solver_3_stepzDPMSolver.dpm_solver_3_step  s   #+B	QJ)UAq9YaZaZB26.."22S88 HHY"bA	B26.."22S884::b>RRTW;UZ\_`Z`YgYgYimorsmsYtwxYx;y  ~D  GJ  ~J  <K  K HHY"bA	$**V$qwwy0366F9Kb9PTUT[T[T]`aTadeTe9fjpsvjv9wwI~r   c           	      l   |'t        || j                  j                  dd             n|}||kD  s|rt        d      t	        j
                  |dz        dz   }t        j                  |||dz   |j                        }	|dz  dk(  rdg|dz
  z  ddgz   }
ndg|dz
  z  |dz  gz   }
t        t        |
            D ]v  }i }|	|   |	|dz      }}|rt        | j                  |      | j                  |      |      \  }}t        j                  || j                  |            }| j                  |      dz  | j                  |      dz  z
  d	z  }n|d
}}| j                  |d||      \  }}|| j                  |      |z  z
  }| j                   | j!                  |||	|   ||d       |
|   dk(  r| j#                  ||||      \  }}n9|
|   dk(  r| j%                  ||||      \  }}n| j'                  ||||      \  }}|||z   || j                  |      | j                  |            z  z   }y |S )Nr_   r   "eta must be 0 for reverse sampling   r   r   r   r-   r?   r6   r   )r   r   r4   t_uprK   r*  )r`   r   r   r|   r%   floorr   r   r   r   rz   rS   rJ   minimumr4   r   r#  r1  r7  r>  )r   r   t_startt_endnferP   r   r   mtsordersr   r*  r4   r/  sdsut_next_r   rK   s                       r   dpm_solver_fastzDPMSolver.dpm_solver_fast  sI   \i\q-adoo6I6I&RV6WX  xEw3ABBJJsQw!#^^GUAE!((C7a<SAE]aV+FSAE]cAgY.Fs6{#AI1r!a%yvA+DJJqM4::f;MsSB--tvvbz:jj(A-

70Cq0HHSP$b!XXi1=NC4::a=3..H!!-""AW_#`aayA~#55aGy5Y9a#55aGy5Y9#55aGy5Y9BL=A

6@R#SSSA- $0 r   c           
         |'t        || j                  j                  dd             n|}|dvrt        d      ||kD  }|s|rt        d      t	        |      |rdndz  }t        j                  |      }t        j                  |      }|}|}d}t        |||	|
|rd	n||      }d
d
d
d
d}|r
||dz
  k  rn	||dz   kD  ri }|r#t        j                  |||j                  z         n"t        j                  |||j                  z         }|rt        | j                  |      | j                  |      |      \  }}t        j                  || j                  |            }| j                  |      dz  | j                  |      dz  z
  dz  }n|d}}| j                  |d||      \  }}|| j                  |      |z  z
  }|dk(  r1| j                  ||||      \  }}| j!                  ||||      \  }}n1| j!                  |||d|      \  }}| j#                  ||||      \  }}t        j                  ||t        j                  |j	                         |j	                               z        }t
        j$                  j'                  ||z
  |z        |j)                         dz  z  }|j+                  |      }|rB|}|||z   || j                  |      | j                  |            z  z   }|}|dxx   dz  cc<   n|dxx   dz  cc<   |dxx   |z  cc<   |dxx   dz  cc<   | j,                  +| j-                  ||d   dz
  |||||j                  d|       |r||dz
  k  r||fS ||dz   kD  r||fS )Nr_   r   >   r-   rA  zorder should be 2 or 3r@  r   r   Tg      ?r   )stepsrH  n_acceptn_rejectr8   r-   r?   r6   r   rC  UUUUUU?)r4  r*  rR  rS  rH  rQ  )r   r   r4   rB  rK   r  r   )r`   r   r   r|   rA   r   tensorr
  rE  r   maximumrS   rJ   r4   r   r1  r7  r>  linalgnormnumelr  r#  ) r   r   rF  rG  r   rtolatolh_initr  r  r  r  rP   r   r   forwardrk   x_prevr  pidinfor*  r4   rL  rM  t_r   rK   x_lowx_highdeltar  s                                    r   dpm_solver_adaptivezDPMSolver.dpm_solver_adaptive6  s   \i\q-adoo6I6I&RV6WX  xE566'/3ABBVW"5||D!||D!#FFFF3CTY[hi1!C")a%$,q54</?I3:eQY/eUVY\Y^Y^U^@_A+DJJqM4::a=#NB]]5$&&*5jjmq(4::b>Q+>>3FBB!XXi1=NC4::a=3..Hz#'#9#9!Qi#9#X y$($:$:1ay$:$Y!	#'#9#9!QuXa#9#b y$($:$:1ay$:$Y!	MM$u}}UYY[&**,/W(WXELL%%uv~&>?!'')sBRRE%%e,FR'\M$**Q-TU,WWWZ A% Z A% K5 KMQM!!-""g1BTUckv{  CF  CH  CH  $Q  LP  $Q  RC #*a%$,F $wG 0154</?F $wr   NNNr   )r?   N)rT  UUUUUU?N)r6   r)   N)rA  皙?q?rh  r6   r)   r6   r  r6   r)   N)r   r   r   r   r   r4   rJ   r   r1  r7  r>  rO  re  __classcell__)r$  s   @r   r  r    s4    ;+,%N3r   r  c           
      v   |dk  s|dk  rt        d      t        ||      5 }t        | ||j                        fd_        j                  |j                  t        j                  |            j                  t        j                  |            |||	|
      cddd       S # 1 sw Y   yxY w)zHDPM-Solver-Fast (fixed step size). See https://arxiv.org/abs/2206.00927.r   %sigma_min and sigma_max must not be 0)totalr   r"  Nc                 f     j                  | d         j                  | d         d|       S Nr4   rB  )rJ   r   rJ   r`  r   
dpm_solvers    r   r>   z!sample_dpm_fast.<locals>.<lambda>t  _    HzGWGWX\]`XaGbq{  rB  rB  CG  HN  CO  rP  >Y  TX  >Y  5Zr   )	r|   r   r  updater#  rO  r4   r   rU  )r   r   r   r   r   r   r   r   rP   r   r   pbarrs  s         `     @r   sample_dpm_fastrw  l  s     A~a@AA	Aw	'4ujt{{K
 (ZJ$))!Z\\%,,y:Q-RT^T`T`afamamnwaxTy{|  B  DK  MZ  [	 
(	'	's   B B//B8c                    |dk  s|dk  rt        d      t        |      5 }t        | ||j                        fd_        j                  |j                  t        j                  |            j                  t        j                  |            |||	|
|||||||      \  }}ddd       |r|fS |S # 1 sw Y   xY w)zPDPM-Solver-12 and 23 (adaptive step size). See https://arxiv.org/abs/2206.00927.r   rl  r   rn  Nc                 f     j                  | d         j                  | d         d|       S rp  rq  rr  s    r   r>   z%sample_dpm_adaptive.<locals>.<lambda>  rt  r   )	r|   r   r  ru  r#  re  r4   r   rU  )r   r   r   r   r   r   r   r   rZ  r[  r\  r  r  r  r  rP   r   r   return_inforv  r`  rs  s        `               @r   sample_dpm_adaptiver{  x  s    A~a@AA	g	$ujt{{K
 (ZJ$00JLLiAX4Y[e[g[ghmhthtu~h  \A  CH  JN  PT  V\  ^d  fl  nt  vC  EH  JQ  S`  a4	 

 $wH 
	s   BB??Cc	                    t        | j                  j                  j                  t        j                  j                        rt        | ||||||||	      S 	 |i n|}|j                  dd       }	|t        ||	      n|}|j                  |j                  d   g      }
d }d }t        t        |      dz
  |      D ]  } | |||   |
z  fi |}t        ||   ||dz      |      \  }}| |||||   ||   |d	       |dk(  r!t        |||   |      }|||   z
  }|||z  z   }n |||          ||      }}d
}||z
  }|||z  z   } ||       ||      z  |z  | |z  j                         |z  z
  } | | ||      |
z  fi |} ||       ||      z  |z  | j                         |z  z
  }||dz      dkD  s| |||   ||dz            |z  |z  z   } |S )Nr_   r   r   c                 >    | j                         j                         S r   r(  r4   s    r   r>   z+sample_dpmpp_2s_ancestral.<locals>.<lambda>      r   c                 >    | j                         j                         S r   r&   r   rq  s    r   r>   z+sample_dpmpp_2s_ancestral.<locals>.<lambda>      *r   r   r   r   r   r?   )rw   r   r   r   r   sample_dpmpp_2s_ancestral_RFr   r`   r   r{   r   rz   rS   rL   r0   )r   r   r"   r   r   r   rP   r   r   r_   r   sigma_fnt_fnr   rK   rR   rQ   r   r   r4   r/  rr   rk   r   r   s                             r   sample_dpmpp_2s_ancestralr    s.   %##//>>@T@T@Z@Z[+E1fj(T[]`bikxyyF!)zJ>>&$'D;H;P)!$7VcM::qwwqzl#D&H*DCK!OW55F1I,;
;1&)VAE]PST
H11vayvay^fgh?Qq	8,AfQi'BAF
A VAYj)9vAA
AAE	AA;!,1aR!VNN4Dx4OOCsHQK$$6E*EJ&!HQK/14zz|j7PPA!a%=1M&)VAE];gEPPA+ 6, Hr   c	           	         |i n|}|j                  dd      }	|t        ||	      n|}|t        | j                  j                  j                  d      dd      z  }|j                  |j                  d   g      }
d }d	 }t        t        |      d
z
  |      D ]J  } | |||   |
z  fi |}d
||d
z      ||   z  d
z
  |z  z   }||d
z      |z  }d
||d
z      z
  }d
|z
  }||d
z      dz  |dz  |dz  z  |dz  z  z
  dz  }| |||||   ||   |d       ||d
z      dk(  r!t        |||   |      }|||   z
  }|||z  z   }nn||   dk(  rd}n* |||          ||      }}d}||z
  }|||z  z   } ||      }|||   z  }||z  d
|z
  |z  z   } | |||
z  fi |}|||   z  }||z  d
|z
  |z  z   }||d
z      dkD  s#|dkD  s*||z  |z   |||   ||d
z            |z  |z  z   }M |S )<Ancestral sampling with DPM-Solver++(2S) second-order steps.Nr_   r   r   r   r)   r   c                 .    | j                         dz   dz  S r   r'   )lbdas    r   r>   z.sample_dpmpp_2s_ancestral_RF.<locals>.<lambda>  s    TXXZ!^2r   c                 .    d| z
  | z  j                         S r   r&  rq  s    r   r>   z.sample_dpmpp_2s_ancestral_RF.<locals>.<lambda>  s    %335r   r   r   r-   r?   r   gH.?)r   r`   r   r   r   r   r   r{   r   rz   rL   ) r   r   r"   r   r   r   rP   r   r   r_   r   r  	lambda_fnr   rK   r   rR   r   r   r   r   r   sigma_st_it_downr  r   rk   sigma_s_i_ratiouD_ir   s                                    r   r  r    s    ")zJ>>&$'D;H;P)!$7VcM 1 1 ? ? P PQa bdqsvwwG::qwwqzl#D2H5I CK!OW55F1I,;
;fQqSk&)3a73>>AaC[>1
qsO	^
!a*a-	1*DZQR]*RRUXX11vayvay^fgh!a%=AQq	8,AfQi'BAF
A ayC 'q	2Ij4IVSL!a%K"1+%q	1O!#q?':h&FFA7T>8Z8C!+fQi!7"Q&!.@*@C)GGA !a%=1q:%*]6!9fQQRUm-TW^-^an-nnAE 6H Hr   c
           	         t        |      dk  r|S |i n|}||dkD     j                         |j                         }}
|j                  dd      }|t	        ||
||d      n|}|j                  |j                  d   g      }| j                  j                  j                  d      }t        t        |      }t        t        |      }t        ||      }|t        |d	d
      z  }t        t        |      dz
  |      D ]R  } | |||   |z  fi |}| |||||   ||   |d       ||dz      dk(  r|}8 |||          |||dz            }}||z
  }||	|z  z   }dd|	z  z  } ||      }||   |j!                         z  }||j!                         z  }||dz      |j!                         z  }t#        |j%                         j!                         |j%                         j!                         |      \  }}|j'                         j%                         }||z
  }||z  | j!                         z  |z  || j)                         z  |z  z
  } |dkD  r|dkD  r| | |||   |      z  |z  |z  z   }  | | ||z  fi |}!t#        |j%                         j!                         |j%                         j!                         |      \  }}|j'                         j%                         }"|"|z
  }d|z
  |z  ||!z  z   }#||z  | j!                         z  |z  || j)                         z  |#z  z
  }|dkD  s.|dkD  s5|| |||   ||dz            z  |z  |z  z   }U |S )zDPM-Solver++ (stochastic).r   Nr   r_   Tr_   rU   r   r   r   r)   r   r   r-   )rz   r;   r<   r   r   r   r{   r   r   r   r   r   r   r   r   r   r'   rS   r   r&   r0   )$r   r   r"   r   r   r   rP   r   r   r  r   r   r_   r   r   r  r  r   rK   lambda_slambda_tr   
lambda_s_1fac	sigma_s_1alpha_s	alpha_s_1alpha_trL  rM  lambda_s_1_h_r   r   	lambda_t_
denoised_ds$                                       r   sample_dpmpp_sder    s    6{a!)zJ!&1*-113VZZ\yI>>&$'D^k^s,Q	94UYZ  zGM::qwwqzl#D&&44EEFVWN,^LH-nMI'?FsCCGCK!OW55F1I,;
;11vayvay^fgh!a%=AA "+6!9!5yA7OhH8#A!AE)Jq1u+C ,IQi(,,.0G!JNN$44IQUmhlln4G ((:(:(<jnn>N>R>R>TVYZFB&&(,,.Kx'Bw&B3))+59I"<UX`<``CQw7Q;IfQi(KKgUXZZZsI$4C
CJ ((:(:(<hlln>P>P>RTWXFBIX%Bc'X-j0@@J7"siik1A5B3++-8OR\8\\AQw7Q;-q	6!a%="IIGSVXXXI 6J Hr   c           	      H   |i n|}|j                  |j                  d   g      }d }d }d}	t        t        |      dz
  |      D ]  }
 | |||
   |z  fi |}| |||
||
   ||
   |d        |||
          |||
dz            }}||z
  }|	||
dz      dk(  r* ||       ||      z  |z  | j	                         |z  z
  }nY| |||
dz
           z
  }||z  }ddd|z  z  z   |z  dd|z  z  |	z  z
  } ||       ||      z  |z  | j	                         |z  z
  }|}	 |S )	DPM-Solver++(2M).Nr   c                 >    | j                         j                         S r   r(  r~  s    r   r>   z!sample_dpmpp_2m.<locals>.<lambda>   r  r   c                 >    | j                         j                         S r   r  rq  s    r   r>   z!sample_dpmpp_2m.<locals>.<lambda>!  r  r   r   r   r   r-   )r   r{   r   rz   r0   )r   r   r"   r   r   r   r   r  r  old_denoisedr   rK   r4   r/  r   h_lastr  r  s                     r   sample_dpmpp_2mr    sp    ")zJ::qwwqzl#D&H*DLCK!OW5F1I,;
;11vayvay^fghOT&Q-%86QJ6!a%=A#5&!HQK/14zz|h7NNAfQUm,,F
Aa1q5k/X5a!e8TTJ&!HQK/14zz|j7PPA 6 Hr   c
           	      X   t        |      dk  r|S |	dvrt        d      |i n|}|j                  dd      }
||dkD     j                         |j	                         }}|t        ||||
d      n|}|j                  |j                  d   g      }| j                  j                  j                  d	      }t        t        |
      }t        ||      }|t        |dd      z  }d}d\  }}t        t        |      dz
  |      D ]  } | |||   |z  fi |}| |||||   ||   |d       ||dz      dk(  r|}ng |||          |||dz            }}||z
  }||dz   z  }||dz      |j!                         z  }||dz      ||   z  | |z  j!                         z  |z  || j#                         j%                         z  |z  z   }||||z  }|	dk(  r9||| j#                         j%                         | z  dz   z  d|z  z  ||z
  z  z   }n9|	dk(  r4|d|z  | j#                         j%                         z  d|z  z  ||z
  z  z   }|dkD  rY|dkD  rT| |||   ||dz            ||dz      z  d|z  |z  j#                         j%                         j'                         z  |z  z   }|}|} |S )zDPM-Solver++(2M) SDE.r   >   heunmidpointz(solver_type must be 'heun' or 'midpoint'Nr_   r   Tr  r   r  r   r)   NNr   r   r  r  r?   )rz   r|   r   r;   r<   r   r   r{   r   r   r   r   r   r   r   r   r'   r0   r   r.   )r   r   r"   r   r   r   rP   r   r   solver_typer_   r   r   r   r   r  r  r   r  r   rK   r  r  h_etar  r  s                             r   sample_dpmpp_2m_sder  5  s    6{a..GHH!)zJ>>&$'D!&1*-113VZZ\yI^k^s,Q	94UYZ  zGM::qwwqzl#D&&44EEFVWN-nMI'?FsCCGLIAvCK!OW55F1I,;
;11vayvay^fgh!a%=AA "+6!9!5yA7OhH8#AqMEQUmhlln4Gq1uq	)aR#XNN,<<q@7uf^^M]MaMaMcCcfnCnnA'QJ&(G~~'7';';'=%'H1'LMQRUVQVW[cfr[rssA J.C'MeVNN,<,@,@,BBa!eLPX[gPghhAQw7Q;fQiA?&Q-OSUXYSY\_S_RfRfRhRlRlRnRsRsRuux9 6: Hr   c
                 ,    t        | |||||||||	
      S )Nr   r   r   rP   r   r   r  )r  
r   r   r"   r   r   r   rP   r   r   r  s
             r   sample_dpmpp_2m_sde_heunr  l  s7    uaJQYcjps  ~E  Ub  p{  |  |r   c	           	         t        |      dk  r|S |i n|}|j                  dd      }	||dkD     j                         |j                         }}
|t	        ||
||	d      n|}|j                  |j                  d   g      }| j                  j                  j                  d      }t        t        |      }t        ||      }|t        |d	d
      z  }d\  }}d\  }}}t        t        |      dz
  |      D ]  } | |||   |z  fi |}| |||||   ||   |d       ||dz      dk(  r|}n |||          |||dz            }}||z
  }||dz   z  }||dz      |j                         z  }||dz      ||   z  | |z  j                         z  |z  || j!                         j#                         z  |z  z   }|w||z  }||z  }||z
  |z  }||z
  |z  }|||z
  |z  ||z   z  z   }||z
  ||z   z  }|j#                         j!                         |z  dz   } | |z  dz
  }!||| z  |z  z   ||!z  |z  z
  }n>|<||z  }"||z
  |"z  }#|j#                         j!                         |z  dz   } ||| z  |#z  z   }|dkD  rY|dkD  rT| |||   ||dz            ||dz      z  d|z  |z  j!                         j#                         j%                         z  |z  z   }||}}||}} |S )zDPM-Solver++(3M) SDE.r   Nr_   r   Tr  r   r  r   r)   r  rf  r   r   r?   r  )rz   r   r;   r<   r   r   r{   r   r   r   r   r   r   r   r   r'   r0   r   r.   )$r   r   r"   r   r   r   rP   r   r   r_   r   r   r   r   r  
denoised_1r   r   h_1h_2r   rK   r  r  r  r  r0r4  d1_0d1_1d1d2phi_2phi_3r  r   s$                                       r   sample_dpmpp_3m_sder  q  su    6{a!)zJ>>&$'D!&1*-113VZZ\yI^k^s,Q	94UYZ  zGM::qwwqzl#D&&44EEFVWN-nMI'?FsCCG'J
"KAsCCK!OW55F1I,;
;11vayvay^fgh!a%=AA!*6!9!5yA7OhH8#AqMEQUmhlln4Gq1uq	)aR#XNN,<<q@7uf^^M]MaMaMcCcfnCnnA1W1W :-3"Z/25TD[B."r'::Tkb2g.		))+e3a7+5B..'E/R1GG!G
*a/		))+e3a75A--Qw7Q;fQiA?&Q-OSUXYSY\_S_RfRfRhRlRlRnRsRsRuux!):J
cSM 6N Hr   c	                     t        |      dk  r|S |i n|}||dkD     j                         |j                         }
}	| t        ||	|
|j	                  dd       d      n|}t        | ||||||||	      S )Nr   r   r_   Fr  r   r   r   rP   r   r   )rz   r;   r<   r   r   r  )r   r   r"   r   r   r   rP   r   r   r   r   s              r   sample_dpmpp_3m_sde_gpur    s    
6{a!)zJ!&1*-113VZZ\yI xE  xM,Q	9:>>Z`bfKgmrs  S`MuaJQYcjps  ~E  Ub  c  cr   c
                     t        |      dk  r|S |i n|}||dkD     j                         |j                         }}
| t        ||
||j	                  dd       d      n|}t        | |||||||||	
      S Nr   r   r_   Fr  r  )rz   r;   r<   r   r   r  r   r   r"   r   r   r   rP   r   r   r  r   r   s               r   sample_dpmpp_2m_sde_heun_gpur    s    
6{a!)zJ!&1*-113VZZ\yI xE  xM,Q	9:>>Z`bfKgmrs  S`M#E1fV^houx  CJ  Zg  u@  A  Ar   c
                     t        |      dk  r|S |i n|}||dkD     j                         |j                         }}
| t        ||
||j	                  dd       d      n|}t        | |||||||||	
      S r  )rz   r;   r<   r   r   r  r  s               r   sample_dpmpp_2m_sde_gpur    s    
6{a!)zJ!&1*-113VZZ\yI xE  xM,Q	9:>>Z`bfKgmrs  S`MuaJQYcjps  ~E  Ub  p{  |  |r   c
                     t        |      dk  r|S |i n|}||dkD     j                         |j                         }}
| t        ||
||j	                  dd       d      n|}t        | |||||||||	
      S )Nr   r   r_   Fr  )r   r   r   rP   r   r   r  )rz   r;   r<   r   r   r  )r   r   r"   r   r   r   rP   r   r   r  r   r   s               r   sample_dpmpp_sde_gpur    s    
6{a!)zJ!&1*-113VZZ\yI xE  xM,Q	9:>>Z`bfKgmrs  S`ME1fh`gmp  {B  R_  cd  e  er   c                    d||z  dz   z  }d||z  dz   z  }||z  }d|z  j                         | d|z
  |z  d|z
  j                         z  z
  z  }|dkD  r,|d|z
  d|z
  z  d|z
  z  j                          |||      z  z  }|S )Nr   r)   r   )r.   )	r   rJ   
sigma_prevnoiser   alpha_cumprodalpha_cumprod_prevalpharB   s	            r   DDPMSampler_stepr    s    %%-1,-MzJ6!;<//E
+			a%i5%8A<M;S;S;U%U!U	VBA~
E	b#556"}:LMSSUXefkmwXxxxIr   c           	         |i n|}|j                  dd       }|t        ||      n|}|j                  |j                  d   g      }	t	        t        |      dz
  |      D ]  }
 | |||
   |	z  fi |}| |||
||
   ||
   |d        ||t        j                  d||
   dz  z         z  ||
   ||
dz      ||z
  ||
   z  |      }||
dz      dk7  sr|t        j                  d||
dz      dz  z         z  } |S )	Nr_   r   r   r   r   r   r)          @)r   r`   r   r{   r   rz   r   r.   )r   r   r"   r   r   r   r   step_functionr_   r   r   rK   s               r   generic_step_samplerr    s4   !)zJ>>&$'D;H;P)!$7VcM::qwwqzl#DCK!OW5F1I,;
;11vayvay^fgh!ejjvayC/?)?@@&)VTUXYTY]]^ai]imstumv\v  yF  G!a%=AC&Q-3"6677A 6 Hr   c           
      .    t        | ||||||t              S r   )r  r  )r   r   r"   r   r   r   r   s          r   sample_ddpmr    s    q&*hQ^`pqqr   c
           	         |i n|}|j                  dd       }
|t        ||
      n|}|j                  |j                  d   g      }t	        dt        |      dz
        }| j                  j                  j                  d      }t        |      }||n
t        |      }t        ||      D ]  } | |||   |z  fi |}| |||||   ||   |d       |}||dz      dkD  s7 |||   ||dz            }|	dkD  r'|	|j                         z  }|j                  | |      }|dkD  r||dz
  z  nd	}|||z
  |z  z   }|d
k7  r||z  }|j                  ||dz      ||      } |S )Nr_   r   r   r   r   r   r   r:   r6   r)   )r   r`   r   r{   r<   rz   r   r   r   r   r   stdr=   noise_scaling)r   r   r"   r   r   r   r   r   s_noise_endnoise_clip_stdr_   r   n_stepsr   s_starts_endr   rK   r  clip_valr4   	s_noise_is                         r   
sample_lcmr    s    ")zJ>>&$'D;H;P)!$7VcM::qwwqzl#D!S[1_%G&&44EEFVWNGnG"*Gk0BEGW-F1I,;
;11vayvay^fgh!a%=1!&)VAE];E!)EIIK7	x@'.{gk"A57?a"77IC	),,VAE]E1EA .  Hr   c
           	      0   |i n|}|j                  |j                  d   g      }
|d   }t        t        |      dz
  |      D ]  }|||   cxk  r|k  rn nt	        |t        |      dz
  z  d      nd}t        j                  |      |	z  }||   |dz   z  }|dkD  r|||dz  ||   dz  z
  dz  z  z   } | |||
z  fi |}t        |||      }| |||||   ||d	       ||dz      |z
  }||dz      |k(  r	|||z  z   }||dz      |k(  r[|||z  z   } | |||dz      |
z  fi |}t        |||dz      |      }d|d   z  }||dz      |z  }d|z
  }||z  ||z  z   }|||z  z   }#|||z  z   } | |||dz      |
z  fi |}t        |||dz      |      }||dz      ||dz      z
  }|||z  z   } | |||dz      |
z  fi |}t        |||dz      |      }d
|d   z  }||dz      |z  }||dz      |z  }d|z
  |z
  }||z  ||z  z   ||z  z   }|||z  z   } |S )Nr   r   r   r   r   r6   r-   r?   r   rA  r   )r   r   r"   r   r   r   r   r   r   r   r   r  r   r   r   r   rK   r   r   r   r   r   r   w2w1r   r   r=  
denoised_3d_3w3s                                  r   sample_heunpp2r    s    ")zJ::qwwqzl#D2JECK!OW55BHFSTIB_Y_B_Gs6{Q/>egq!G+1I+	19C9>F1IN:sBBBAI,;
;Ix(11vayy^fghAE]Y&!a%=E!AF
AAE]e# a"f*CsF1q5MD$8GJGJsF1q5M:6CF1IA!QBRB"fsRx'G GbL A a"f*CsF1q5MD$8GJGJsF1q5M:6C!a%=6!a%=0Dd
"CsF1q5MD$8GJGJsF1q5M:6CF1IAA"BA"BR"B1frCx'"s(2GGbL A_ 6` Hr   c           	         |i n|}|j                  |j                  d   g      }|}g }	t        t        |      dz
  |      D ]#  }
||
   }||
dz      }|} | |||z  fi |}| |||
||
   ||
   |d       ||z
  |z  }t	        ||
dz         }|dk(  r|}n|dk(  r|||z
  |z  z   }nw|dk(  r|||z
  d|z  |	d   z
  z  dz  z   }nZ|dk(  r$|||z
  d|z  d	|	d   z  z
  d
|	d   z  z   z  dz  z   }n1|dk(  r,|||z
  d|z  d|	d   z  z
  d|	d   z  z   d|	d   z  z
  z  dz  z   }t        |	      |dz
  k(  r%t        |dz
        D ]  }|	|dz      |	|<    ||	d<   |	j                  |       & |S )Nr   r   r   r   r-   rA  r            r        7   ;   %   	      )r   r{   r   rz   r;   r   r   )r   r   r"   r   r   r   	max_orderr   x_nextbuffer_modelr   t_curr/  x_currK   d_curr   r   s                     r   sample_ipndmr   S  s    !)zJ::qwwqzl#DFLCK!OW55q	A;
;11vayvay^fgh!U*Iqs#Q;FaZfun55FaZfunU\"=M1MNQRRRFaZfuneb<PRCS>S1SVWZfgiZjVj1jknpppFaZfuneb<PRCS>S1SVX[ghj[kVk1knor~  @B  sC  oC  2C  D  GI  I  IF|	A-9q=)".qs"3Q *$L&; 6> Mr   c           	         |i n|}|j                  |j                  d   g      }|}|}	g }
t        t        |      dz
  |      D ]  }||   }||dz      }|} | |||z  fi |}| |||||   ||   |d       ||z
  |z  }t	        ||dz         }|dk(  r|}nL|dk(  r|||z
  |z  z   }n:|dk(  r=||z
  }||	|dz
     z
  }d||z  z   dz  }||z   dz  }|||z
  ||z  ||
d   z  z   z  z   }n|dk(  r||z
  }||	|dz
     z
  }|	|dz
     |	|dz
     z
  }d|d||z   z  z  |||z   z  z  |||z   z  z  z
  dz  }d||z  z   dz  |z   }||z   dz  d||z  z   |z  z
  }||z  |z  }|||z
  ||z  ||
d   z  z   ||
d   z  z   z  z   }nb|d	k(  r\||z
  }||	|dz
     z
  }|	|dz
     |	|dz
     z
  }|	|dz
     |	|dz
     z
  }d|d||z   z  z  |||z   z  z  |||z   z  z  z
  dz  }d|d||z   z  z  z
  dz  d|d||z   z  z  z
  |z  d
||z   |z   z  z  z   |||z   z  ||z   |z   z  z  |||z   z  ||z   |z   z  z  }d||z  z   dz  |z   |z   }||z   dz  d||z  z   |z  z
  d||z  z   |||z   z  |||z   z  z  z   |z  z
  }||z  |z  ||z  |||z   z  |||z   z  z  d||z  z   z  z   |z  z   }| |||z   z  |||z   z  z  z  |z  |z  }|||z
  ||z  ||
d   z  z   ||
d   z  z   ||
d   z  z   z  z   }t        |
      |dz
  k(  r3t        |dz
        D ]  }|
|dz      |
|<    |j                         |
d<   |
j                  |j                                 |S )Nr   r   r   r   r-   r   rA  r  r     r  )r   r{   r   rz   r;   r   r}   r   )r   r   r"   r   r   r   r  r   r  t_stepsr  r   r  r/  r  rK   r  r   h_nh_n_1coeff1coeff2h_n_2tempcoeff3h_n_3temp1temp2coeff4r   s                                 r   sample_ipndm_vr  ~  s   !)zJ::qwwqzl#DFGLCK!OW55q	A;
;11vayvay^fgh!U*Iqs#Q;FaZfun55FaZE>CWQqS\)E3;'1,FU{^a'Ffun%&<XZK[B[1[\\FaZE>CWQqS\)EQqS\GAaCL0EqC%K01SC%K5HIUV[^cVcMdeeijjD3;'1,t3FU{^a'1uu}+<*DDFE\E)Ffun%&<XZK[B[1[^dgstvgw^w1wxxFaZE>CWQqS\)EQqS\GAaCL0EQqS\GAaCL0ES5[ 12cS5[6IJeW\_dWdNeffjkkE#cEk!233q8AqCRWKGX@Y<Y]`;`deilotitw|i|d};~~S5[)S5[5-@ACFKuW\}F]afinanqvavFwyE3;'1,u4u<FU{^a'1uu}+<*EEeV[mI\`einqviv`w  |A  EJ  MR  ER  |S  aT  JU  Y^  I^  ^FU]U*uu}%RW-AX\aejmrer\sAtyz  ~C  FK  ~K  zK  AL  /L  PU  .U  UFVu6%55=:QRSV[[^ccFfun%&<XZK[B[1[^dgstvgw^w1w  {A  DP  QS  DT  {T  2T  U  UF|	A-9q=)".qs"3Q *$||~L/g 6j Mr   c           	      H   |i n|}|j                  |j                  d   g      }|}	|}
t        j                  |
||      }g }t	        t        |      dz
  |      D ]A  }||   }||dz      }|	} | |||z  fi |}| |||||   ||   |d       ||z
  |z  }t        ||dz         }|dk  rd}|dk(  r|||z
  |z  z   }	nz|dk(  r||   \  }}|||z  z   ||d   z  z   }	n[|dk(  r$||   \  }}}|||z  z   ||d   z  z   ||d	   z  z   }	n2|d
k(  r-||   \  }}}}|||z  z   ||d   z  z   ||d	   z  z   ||d   z  z   }	t        |      |dz
  k(  r3t        |dz
        D ]  }||dz      ||<    |j                         |d<   #|j                  |j                                D |	S )Nr   )	deis_moder   r   r   r-   r   rA  r  r  r  )
r   r{   r	   get_deis_coeff_listr   rz   r;   r   r}   r   )r   r   r"   r   r   r   r  r  r   r  r  
coeff_listr  r   r  r/  r  rK   r  r   	coeff_curcoeff_prev1coeff_prev2coeff_prev3r   s                            r   sample_deisr    sG   !)zJ::qwwqzl#DFG))'9	RJLCK!OW55q	A;
;11vayvay^fgh!U*Iqs#Q;EA:fun55FaZ%/]"I{Y..|B?O1OOFaZ2<Q-/I{KY..|B?O1OOR]`lmo`pRppFaZ?I!}<I{KY..|B?O1OOR]`lmo`pRpps~  BN  OQ  BR  tR  RF|	A-9q=)".qs"3Q *$||~L/C 6F Mr   c	           	         |i n|}|j                  dd      }	|t        ||	      n|}| j                  j                  j	                  d      }
t        t        |
      }|t        |
dd      z  }dfd}|j                  d	i       j                         }t        j                  j                  ||d
      |d	<   |j                  |j                  d   g      }t        t        |      dz
  |      D ]  } | |||   |z  fi |}| |||||   ||   |d       ||dz      dk(  r|}7||    |||         j                         z  }||dz       |||dz            j                         z  }t!        |||   |z        }t#        ||   |z  ||dz      |z  |      \  }}||z  }||z  ||z  z   }|dkD  s|dkD  s|| |||   ||dz            z  |z  |z  z   } |S )z3Ancestral sampling with Euler method steps (CFG++).Nr_   r   r   r  r   r)   c                     | d   | d   S Nuncond_denoisedrK   re   r,  r  s    r   post_cfg_functionz8sample_euler_ancestral_cfg_pp.<locals>.post_cfg_function      01Jr   model_optionsTdisable_cfg1_optimizationr   r   r   r   r   )r   r`   r   r   r   r   r   r   copyr   #set_model_options_post_cfg_functionr   r{   r   rz   r'   rL   rS   )r   r   r"   r   r   r   rP   r   r   r_   r   r  r  r   r   r   rK   r  r  r   rR   rQ   r  s                         @r   sample_euler_ancestral_cfg_ppr%    s5    ")zJ>>&$'D;H;P)!$7VcM&&44EEFVWN-nMIsCCGO 
 NN?B7<<>M"'"5"5"Y"YZgiz  W["Y  #\J::qwwqzl#DCK!OW5F1I,;
;11vayvay^fgh!a%=AAQi)F1I"6":":"<<GQUmiq1u&>&B&B&DDGQq	7_#<=A $6fQi'6I6RSVWRW=[bKbhk#l J :-J ("Z!^3AQw7Q;-q	6!a%="IIGSV^^^' 6( Hr   c                 *    t        | |||||ddd	      S )zEuler method steps (CFG++).r6   Nr  )r%  )r   r   r"   r   r   r   s         r   sample_euler_cfg_ppr'    s2     )6j[cmtz}  HK  [_  `  `r   c	           	      .   |i n|}|j                  dd      }	|t        ||	      n|}|t        | j                  j                  j                  d      dd      z  }dgfd}
|j                  d	i       j                         }t        j                  j                  ||
d
      |d	<   |j                  |j                  d   g      }d }d }t        t        |      dz
  |      D ](  } | |||   |z  fi |}t        ||   ||dz      |      \  }}| |||||   ||   |d       |dk(  rt        |||   d         }|||z  z   }n |||          ||      }}d}||z
  }|||z  z   } ||       ||      z  ||d   z
  z   z  | |z  j                         |z  z
  } | | ||      |z  fi |} ||       ||      z  ||d   z
  z   z  | j                         |z  z
  }||dz      dkD  s| |||   ||dz            |z  |z  z   }+ |S )r  Nr_   r   r   r   r)   r   c                     | d   d<   | d   S )Nr  r   rK   re   )r,  r	  s    r   r  z;sample_dpmpp_2s_ancestral_cfg_pp.<locals>.post_cfg_function-  s    ()QJr   r   Tr!  c                 >    | j                         j                         S r   r(  r~  s    r   r>   z2sample_dpmpp_2s_ancestral_cfg_pp.<locals>.<lambda>5  r  r   c                 >    | j                         j                         S r   r  rq  s    r   r>   z2sample_dpmpp_2s_ancestral_cfg_pp.<locals>.<lambda>6  r  r   r   r   r   r   r?   )r   r`   r   r   r   r   r#  r   r$  r   r{   r   rz   rS   rL   r0   )r   r   r"   r   r   r   rP   r   r   r_   r  r   r   r  r  r   rK   rR   rQ   r   r4   r/  r  r   rk   r   r   r	  s                              @r    sample_dpmpp_2s_ancestral_cfg_ppr,  $  s    ")zJ>>&$'D;H;P)!$7VcM 1 1 ? ? P PQa bdqsvwwG3D  NN?B7<<>M"'"5"5"Y"YZgiz  W["Y  #\J::qwwqzl#D&H*DCK!OW55F1I,;
;1&)VAE]PST
H11vayvay^fgh?Qq	47+A1z>)A VAYj)9vAA
AAE	AA;!,ha6H1IJqbSTf^^M]`hMhhCsHQK$$6E*EJ&!HQK/ADG9K4LMRSQSPZPZP\_iPiiA!a%=1M&)VAE];gEPPA+ 6, Hr   c           	         |i n|}|j                  |j                  d   g      }d }d}dfd}	|j                  di       j                         }
t        j
                  j                  |
|	d      |d<   t        t        |      dz
  |	      D ]  } | |||   |z  fi |}| |||||   ||   |d
        |||          |||dz            }}||z
  }|||dz      dk(  rt        j                  |        z  }nV| |||dz
           z
  }||z  }t        j                  |        z  t        j                  |       dd|z  z  z  ||z
  z  z
  }||z   t        j                  |       |z  z   }} |S )r  Nr   c                 >    | j                         j                         S r   r  rq  s    r   r>   z(sample_dpmpp_2m_cfg_pp.<locals>.<lambda>U  r  r   c                     | d   | d   S r  re   r  s    r   r  z1sample_dpmpp_2m_cfg_pp.<locals>.post_cfg_functionY  r  r   r   Tr!  r   r   r   r-   )r   r{   r   r#  r   r   r$  r   rz   r   r'   r0   )r   r   r"   r   r   r   r   r  old_uncond_denoisedr  r   r   rK   r4   r/  r   denoised_mixr  r  r  s                      @r   sample_dpmpp_2m_cfg_ppr2  P  s    ")zJ::qwwqzl#D*DO 
 NN?B7<<>M"'"5"5"Y"YZgiz  W["Y  #\JCK!OW5F1I,;
;11vayvay^fghOT&Q-%86QJ&&Q-1*<!IIqbM>O;LfQUm,,F
A!IIqbM>O;ekk1"oQRVWZ[V[Q\>]aila  ?A  AL|#eiima&77- 6 Hr   c
           	        #$ |i n|}|j                  dd       }
|t        ||
      n|}|t        | j                  j                  j                  d      dd      z  }|j                  |j                  d   g      }d }d }d	 ##fd
}d }d }d $$fd}|	rE|j                  di       j                         }t        j                  j                  ||d      |d<   t        t        |      dz
  |      D ]v  } | |||   |z  fi |}t        ||   ||dz      |      \  }}| |||||   ||   |d       |dk(  s|<|	rt        |||   $      }|||z  z   }nt        |||   |      }|||   z
  }|||z  z   }n |||          ||       ||       |||dz
           f\  }}}}||z
  }||z
  |z  } #|        ||       } }t        j                   || |z  z
  d      }!t        j                   | |z  d      }"|	r#||$z
  z   } ||      |z  ||!$z  |"|z  z   z  z   }n ||      |z  ||!|z  |"|z  z   z  z   }||dz      dkD  r| |||   ||dz            |z  |z  z   }|	r$}n|}|}y |S )Nr_   r   r   r   r)   r   c                 >    | j                         j                         S r   r(  r~  s    r   r>   zres_multistep.<locals>.<lambda>x  r  r   c                 >    | j                         j                         S r   r  rq  s    r   r>   zres_multistep.<locals>.<lambda>y  r  r   c                 2    t        j                  |       | z  S r   r   r~  s    r   r>   zres_multistep.<locals>.<lambda>z  s    A*r   c                       |       dz
  | z  S r   re   )r4   phi1_fns    r   r>   zres_multistep.<locals>.<lambda>{  s    c)Q.r   c                     | d   | d   S r  re   r  s    r   r  z(res_multistep.<locals>.post_cfg_function  r  r   r   Tr!  r   r   r   r   r6   nan)r   r`   r   r   r   r   r   r{   r#  r   r$  r   rz   rS   rL   r   
nan_to_num)%r   r   r"   r   r   r   r   r   rP   cfg_ppr_   r   r  r  phi2_fnold_sigma_downr  r  r   r   rK   rR   rQ   r   r   r4   t_oldr/  t_prevr   c2phi1_valphi2_valr  r  r8  r  s%                                      @@r   res_multisteprE  q  s   !)zJ>>&$'D;H;P)!$7VcM 1 1 ? ? P PQa bdqsvwwG::qwwqzl#D&H*D*G.GNLO 
 ";@@B&+&9&9&]&]^km~  [_&]  '`
?#CK!OW55F1I,;
;1&)VAE]PST
H11vayvay^fgh?l2F1I7q:~-F1Ix0&)+BJ (,F1I^8LdS]N^`deklmpqlqer`s's$Auff
A5.A%B!(!gqbkhH!!(X]":DB!!(R-S9BO34QK!Oa2+?"|BS+S&TTQK!Oa2=2;L+L&MM !a%=1M&)VAE];gEPPA*L#L#M 6N Hr   c                 ,    t        | |||||||dd
      S )Nr6   Fr   r   r   r   r   rP   r=  rE  r   r   r"   r   r   r   r   r   s           r   sample_res_multisteprJ    s6    6j8]dnu  FS  Y[  di  j  jr   c                 ,    t        | |||||||dd
      S )Nr6   TrG  rH  rI  s           r   sample_res_multistep_cfg_pprL    s6    6j8]dnu  FS  Y[  dh  i  ir   c	                 ,    t        | ||||||||d
      S )NFrG  rH  	r   r   r"   r   r   r   rP   r   r   s	            r   sample_res_multistep_ancestralrO    s6    6j8]dnu  FS  Y\  ej  k  kr   c	                 ,    t        | ||||||||d
      S )NTrG  rH  rN  s	            r   %sample_res_multistep_ancestral_cfg_pprQ    s6    6j8]dnu  FS  Y\  ei  j  jr   c           	      ^   |i n|}|j                  |j                  d   g      }d}	dfd}
|rE|j                  di       j                         }t        j
                  j                  ||
d      |d<   t        t        |      dz
  |      D ]  } | |||   |z  fi |}|rt        |||         }nt        |||   |      }| |||||   ||   |d	       ||dz      ||   z
  }||dz      dk(  r|}n1|r||||dz      z  z   }n|||z  z   }|dk\  r|dz
  ||	z
  z  }|||z  z   }|}	 |S )
zLGradient-estimation sampler. Paper: https://openreview.net/pdf?id=o2ND9v0CeKNr   c                     | d   | d   S r  re   r  s    r   r  z5sample_gradient_estimation.<locals>.post_cfg_function  r  r   r   Tr!  r   r   r   )
r   r{   r   r#  r   r   r$  r   rz   rL   )r   r   r"   r   r   r   ge_gammar=  r   old_dr  r   r   rK   r   r   d_barr  s                    @r   sample_gradient_estimationrW    s    ")zJ::qwwqzl#DEO 
 ";@@B&+&9&9&]&]^km~  [_&]  '`
?#CK!OW5F1I,;
;Qq	?3AQq	8,A11vayvay^fghAE]VAY&!a%=AA q6!a%=00BJAv!A!e)4
N/ 60 Hr   c           
      (    t        | ||||||d      S )NT)r   r   r   rT  r=  )rW  )r   r   r"   r   r   r   rT  s          r   !sample_gradient_estimation_cfg_pprY    s-    %eQ:X`jq  }E  NR  S  Sr   c
           	         |i n|}|j                  dd      }
|t        ||
      n|}|t        | j                  j                  j                  d      dd      z  }|j                  |j                  d   g      }d }||n|}d	}t        j                  d|t        j                  |j                  
      }| j                  j                  j                  d      }t        ||      }t        ||      }|j                         j                         }d}d}t!        t#        |      dz
  |      D ]  } | |||   |z  fi |}| |||||   ||   |d       t%        |	|dz         }||dz      dk(  r|}nS||   ||dz      }}||   |z  }||dz      |z  }||z  } ||       ||      z  }||z  |z  |d|z
  z  |z  z   }|dk\  r||z
  }| |z  }|||z  z   } ||      } t        j&                  d| z        |z  }!||z
  |||dz
     z
  z  }"||||! ||      z  z   z  |"z  z   }|dk\  rOt        j&                  ||z
  | z        |z  }#|"|z
  |||dz
     z
  dz  z  }$|||dz  dz  |# ||      z  z   z  |$z  z   }|"}|dkD  rK|| |||   ||dz            z  |z  |dz  |dz  |dz  z  z
  j)                         j+                  d      z  z   }|} |S )zExtended Reverse-Time SDE solver (VP ER-SDE-Solver-3). arXiv: https://arxiv.org/abs/2309.06169.
    Code reference: https://github.com/QinpengCui/ER-SDE-Solver/blob/main/er_sde_solver.py.
    Nr_   r   r   r   r)   r   c                 4    | | dz  j                         dz   z  S )Ng333333?g      $@r  r   s    r   default_er_sde_noise_scalerz2sample_er_sde.<locals>.default_er_sde_noise_scaler  s    Q#XNN$t+,,r   g      i@)rW   r   r   r   r   r-   rA  r6   r:  )r   r`   r   r   r   r   r   r{   r   arangefloat32r   r   r   r   r'   r   rz   r;   r  r.   r<  )%r   r   r"   r   r   r   r   r   noise_scaler	max_stager_   r   r\  num_integration_pointspoint_indicer   half_log_snrs
er_lambdasr  old_denoised_dr   rK   
stage_useder_lambda_ser_lambda_tr  r  r_alphar  r   lambda_step_size
lambda_pos
scaled_posrk   r  s_u
denoised_us%                                        r   sample_er_sdero    s   
 ")zJ>>&$'D;H;P)!$7VcM 1 1 ? ? P PQa bdqsvwwG::qwwqzl#D- 3?2F.LL"<<#9WXW_W_`L&&44EEFVWN'?F)&.AM""$((*JLNCK!OW55F1I,;
;11vayvay^fghAE*
!a%=AA'1!}jQ6GKQi+-GQUmk1G'G[)L,EEA !a'QU"3h">>AQ ;.$&3)?#? (<:J+JJ
)*5
 IIa*n-0@@&5+
STWXSXHY:YZ
2L,E(E#EFSS?))Z+%=$KLO__C",~"=;Q[\]`a\aQbCbfgBg!hJGa1}s\+=V7V'VWZdddA!+{-q	6!a%="IIGSWbfgWgjuyzjz}~  CD  ~D  kD  XD  WJ  WJ  WL  WW  WW  \_  WW  W`  `  `K 6L Hr   c           	         |
dvrt        d      |i n|}|j                  dd      }|t        ||      n|}|j                  |j                  d   g      }| j
                  j                  j                  d      }|t        |dd	      z  }|dkD  xr |dkD  }t        t        |
      }t        t        |
      }t        ||      }dd|	z  z  }t        t        |      dz
  |      D ]l  } | |||   |z  fi |}| |||||   ||   |d       ||dz      dk(  r|}8 |||          |||dz            }}||z
  }||dz   z  }t        j                   |||	      } ||      }||j#                         z  }||dz      |j#                         z  }|||   z  |	 |z  |z  j#                         z  |z  |t%        |	 |z        z  |z  z
  }|rMd|	z  |z  |z  j'                         j)                         j+                          |||   |      z  }|||z  |z  z   } | |||z  fi |}|
dk(  rQt        j                   |||      }||dz      ||   z  | |z  j#                         z  |z  |t%        |       z  |z  z
  }nX|
dk(  rSt-        |       |	z  } t%        |       | z
  }!||dz      ||   z  | |z  j#                         z  |z  ||!|z  | |z  z   z  z
  }|s|	dz
  |z  |z  }"|"j#                         z  }||"j/                  d      j'                         j)                         j+                          ||||dz            z  z   }||||dz      z  |z  z   }o |S )zSEEDS-2 - Stochastic Explicit Exponential Derivative-free Solvers (VP Data Prediction) stage 2.
    arXiv: https://arxiv.org/abs/2305.14267 (NeurIPS 2023)
    >   phi_1r  z&solver_type must be 'phi_1' or 'phi_2'Nr_   r   r   r   r   r)   r  r   r-   r   r   r  rq  r  )r|   r   r`   r   r{   r   r   r   r   r   r   r   r   r   rz   r   r   r'   r   r0   r   r.   r   mul)#r   r   r"   r   r   r   rP   r   r   r  r  r_   r   r   inject_noiser  r  r  r   rK   r  r  r   r  r  r  r  r  r   	sde_noiser   r  r  r  segment_factors#                                      r   sample_seeds_2rv  7  s   
 ,,ABB!)zJ>>&$'D;H;P)!$7VcM::qwwqzl#D&&44EEFVWNsCCG7*w{L,^LH-nMI'?F
q1u+CCK!OW55F1I,;
;11vayvay^fgh!a%=AA&vay19VAE]3K(xS1WZZ(A6
Z(	
 00	Q-(,,.0 &)#rAv|&8&8&::Q>ZYZXZ]bXbMcAcfnAnna!c)002668==?-PVWXPY[dBeeI	I-77C3	D 0?J?
 '!Hj#>Jq1uq	)aR#XNN,<<q@7ZY^X^M_C_blCllAG#UF#a'BUF#b(Bq1uq	)aR#XNN,<<q@7bS[m^`cm^mNmCnnA!eq[3.N!N$6$6$88I!N$6$6q$9$?$?$A$E$E$G$L$L$NQ^_hjpqruvqvjwQx$xxIIq1u-77AM 6N Hr   c                 .    t        | |||||dddd|      S )z[Deterministic exponential Heun second order method in data prediction (x0) and logSNR time.r6   Nr)   r   r   r   rP   r   r   r  r  rv  )r   r   r"   r   r   r   r  s          r   sample_exp_heun_2_x0rz  v  s;     %FzH^eknx{  LP  TW  ep  q  qr   c
                 .    t        | ||||||||d|	      S )zXStochastic exponential Heun second order method in data prediction (x0) and logSNR time.r)   rx  ry  r  s
             r   sample_exp_heun_2_x0_sder|  |  s;     %FzH^eknx  P]  ad  r}  ~  ~r   c           	         |i n|}|j                  dd      }|t        ||      n|}|j                  |j                  d   g      }| j                  j
                  j                  d      }|t        |dd      z  }|dkD  xr |dkD  }t        t        |      }t        t        |      }t        ||      }t        t        |      d	z
  |
      D ]  } | |||   |z  fi |}| |||||   ||   |d       ||d	z      dk(  r|}8 |||          |||d	z            }}||z
  }||d	z   z  }t        j                  |||	      }t        j                  |||
      } ||       ||      }}||j!                         z  }||j!                         z  }||d	z      |j!                         z  }|||   z  |	 |z  |z  j!                         z  |z  |t#        |	 |z        z  |z  z
  }|rMd|	z  |z  |z  j%                         j'                         j)                          |||   |      z  }|||z  |z  z   } | |||z  fi |} |
|	z  t+        |
 |z        z  }!t#        |
 |z        |!z
  }"|||   z  |
 |z  |z  j!                         z  |z  ||"|z  |!| z  z   z  z
  }#|rq|	|
z
  |z  |z  }$|$j!                         z  }||$j-                  d      j%                         j'                         j)                          |||      z  z   }|#||z  |z  z   }# | |#||z  fi |}%t+        |       |
z  }&t#        |       |&z
  }'||d	z      ||   z  | |z  j!                         z  |z  ||'|z  |&|%z  z   z  z
  }|s|
d	z
  |z  |z  }$|$j!                         z  }||$j-                  d      j%                         j'                         j)                          ||||d	z            z  z   }||||d	z      z  |z  z   } |S )zSEEDS-3 - Stochastic Explicit Exponential Derivative-free Solvers (VP Data Prediction) stage 3.
    arXiv: https://arxiv.org/abs/2305.14267 (NeurIPS 2023)
    Nr_   r   r   r   r   r)   r  r   r   r   r  r-   )r   r`   r   r{   r   r   r   r   r   r   r   r   r   rz   r   r   r'   r   r0   r   r.   r   rr  )(r   r   r"   r   r   r   rP   r   r   r_1r_2r_   r   r   rs  r  r  r   rK   r  r  r   r  r  
lambda_s_2r  	sigma_s_2r  	alpha_s_2r  r   rt  r   a3_2a3_1r=  ru  r  r  r  s(                                           r   sample_seeds_3r    s   
 ")zJ>>&$'D;H;P)!$7VcM::qwwqzl#D&&44EEFVWNsCCG7*w{L,^LH-nMI'?FCK!OW55F1I,;
;11vayvay^fgh!a%=AA&vay19VAE]3K(xS1WZZ(C8
ZZ(C8
'
3Xj5I9	
 00	
 00	Q-(,,.0 &)#tax#~&:&:&<<q@9z[^Z^afZfOgCgjrCrrcA+22488:??AMRXYZR[]fDggI	I-77C3	D 0?J?
 Sy:sdUl333$,'$.&)#tax#~&:&:&<<q@9PTW_P_bfisbsPsCtt!Ci1_s2N!N$6$6$88I!N$6$6q$9$?$?$A$E$E$G$L$L$NQ^_hjsQt$ttI	I-77C3	D 0?J?
 #%"$1q5MF1I%!c(881<w"x-Z\_iZiJi?jj!Ag]S0N!N$6$6$88I!N$6$6q$9$?$?$A$E$E$G$L$L$NQ^_hjpqruvqvjwQx$xxIIq1u-77A] 6^ Hr   c           
         t        |      dk  r|S |i n|}|j                  dd      }|t        ||      n|}|j                  |j                  d   g      }| j
                  j                  j                  d      }|t        |dd      z  }t        ||      }t        ||	      }|:|j                  d
      }|j                  d      }t        j                  ||d      }t        |	|
      }|}d}d}d}g }|d   j                         dk(  }t!        t        |      dz
  |      D ]  } | |||   |z  fi |}| |||||   ||   |d       |j#                  |       || d }t%        |	t        |            }|dk(  s||dz      dk(  r|sd}nt%        |
t        |            }|r6t%        |t        |      dz
  |z
        }t%        |t        |      dz
  |z
        }|dk(  r|}n|||z
  dz   |dz    }t        j&                  ||   |||dz
     ||   ||d      }t)        j*                  || d d      } t)        j,                  | |dgdgf      }!||   ||dz
     z  |dz   |z  j/                         z  |z  |!z   }|dkD  r
|dkD  r||z   }|r | |||   |z  fi |}||d<   ||dz      dk(  r|} |||dz            }|||z
  dz   |dz    }t        j&                  ||dz      |||   ||dz      ||d      }t)        j*                  || d d      } t)        j,                  | |dgdgf      }"||dz      ||   z
  }||dz      ||   z  |dz   |z  j/                         z  |z  |"z   }|dkD  sM|dkD  sT |||   ||dz            ||dz      z  d|dz  z  |z  j1                         j3                         j5                         z  |z  }||z   } |S )zGStochastic Adams Solver with predictor-corrector method (NeurIPS 2023).r   Nr_   r   r   r   r   r)   r  g?g?r   r6   r   r   r   r-   T)is_corrector_step)dim)dimsFr  )rz   r   r`   r   r{   r   r   r   r   r   r   r   r
   get_tau_interval_funcr<   rv   r   r   r;   !compute_stochastic_adams_b_coeffsr   r   	tensordotr'   r0   r   r.   )#r   r   r"   r   r   r   tau_funcr   r   predictor_ordercorrector_orderuse_pecesimple_order_2r_   r   r   lambdasstart_sigma	end_sigmamax_used_orderx_predr   tau_tr  	pred_listlower_order_to_endr   rK   predictor_order_usedcorrector_order_usedcurr_lambdasb_coeffspred_matcorr_respred_ress#                                      r   sample_sa_solverr    s    6{a!)zJ>>&$'D;H;P)!$7VcM::qwwqzl#D&&44EEFVWNsCCG'?F#F>JG$55c:"33C8	22;	sS/:NFAEEI  *a/CK!OW55T!1@Z@6F1IFSTIcklm"~o./	"?C	NC6fQUmq(#$ #&I#G #&';S[1_q=P#Q #&';S[1_q=P#Q   1$A"1';#;a#?AFL BBq	A
"&H {{9.B-B-C#D!LHx!qc
KHq	F1q5M)uz]Q->,C,C,EEIHTAqyWq[I F1I$4C
C (	" !a%=AFVAE]+E"1';#;a#?AFL BBq1u
A"'H {{9.B-B-C#D!LHx!qc
KHA+AAE]VAY.EQJ-!2C1H1H1JJQNQYYFqyWq[%fQiA?&Q-OSUX]abXbSbefSfRmRmRoRsRsRuRzRzR||  @G  G%K 6L Mr   c                 2    t        | |||||||||	|
d|      S )u`   Stochastic Adams Solver with PECE (Predict–Evaluate–Correct–Evaluate) mode (NeurIPS 2023).T)
r   r   r   r  r   r   r  r  r  r  )r  )r   r   r"   r   r   r   r  r   r   r  r  r  s               r   sample_sa_solver_pecer  .  sM     E1fh`grz  EL  \i  {J  \k  vz  KY  Z  Zr   c           
         |i n|}|j                  di       }|j                  di       }|j                  dk7  r&t        d|j                   d|j                   d      | j                  j                  }	|	j
                  }
t        |
d      rt        |
d	      st        d
      |j                  dd      }|j                  \  }}}}}| dz   | dz   z  }| |z   }|j                  }|	j                         }|
j                  |||z  ||      }|
j                  |||      }t        j                  |      }|j                  |j                  d   g      }d}|j                  di       j                  dd      }|w|	j                  |      j!                  ||      }|j                  d   }||ddddd|f<   d||d}||d<   |j#                  dg      } | |||z  fi |}|}||z
  }| |z   }t%        |      dz
  } || z  }!d}"	 t'        ||      D ],  }#t)        |||z
        }$|||$z   }&}%|dddd|%|&f   }'|||d}||d<   t+        |       D ]  }( | |'||(   |z  fi |})||"| z  |!z  }* ||'|*||(   ||(   |)d       ||(dz      dk(  r|)}'n`||(dz      }+t        j,                  ||#dz  z   |(z          t        j.                  |)      },d|+z
  |)z  |+|,z  z   }'|D ]  }-|-dxx   |$|z  z  cc<    |"dz  }" |'|dddd|%|&f<   |D ]  }-|-dxx   |$|z  z  cc<    |j#                  dg      } | |'||z  fi |}||$z  }/ 	 |j1                  dd       |S # |j1                  dd       w xY w)a  
    Autoregressive video sampler: block-by-block denoising with KV cache
    and flow-match re-noising for Causal Forcing / Self-Forcing models.

    Requires a Causal-WAN compatible model (diffusion_model must expose
    init_kv_caches / init_crossattn_caches) and 5-D latents [B,C,T,H,W].

    All AR-loop parameters are passed via the SamplerARVideo node, not read
    from the checkpoint or transformer_options.
    Nr   transformer_optionsr  z=ar_video sampler requires 5-D video latents [B,C,T,H,W], got z-D tensor with shape zU. This sampler is only compatible with autoregressive video models (e.g. Causal-WAN).init_kv_cachesinit_crossattn_cachesu   ar_video sampler requires a Causal-WAN compatible model whose diffusion_model exposes init_kv_caches() and init_crossattn_caches(). The loaded checkpoint does not support this interface — choose a different sampler.r_   r   r-   	ar_configinitial_latentr   )start_frame	kv_cachescrossattn_cachesar_stater   r   r   i  r)   end)r   rI   r|   r{   r   diffusion_modelhasattr	TypeErrorr   	get_dtyper  r  r   rs   r   process_latent_inr   r   rz   r   r;   r   r^   r   rp   ).r   r   r"   r   r   r   num_frame_per_blockr   r  r   causal_modelr_   bsclat_tlat_hlat_wframe_seq_len
num_blocksr   model_dtyper  r  outputr   current_start_framer  n_initr  
zero_sigma_	remainingnum_sigma_stepstotal_real_steps
step_count	block_idxbffsfenoisy_inputr   rK   scaled_ir\   fresh_noisecaches.                                                 r   sample_ar_videor  4  s\    ")zJNN?B7M'++,A2Fvv{KAFF8Shijipiphq rb b
 	

 ##//K..LL"23Ne8fN
 	
 >>&!$D!"B5%fkNv{^3M6001JXXF'')K++B0Ev{[I#99"fkRa F::qwwqzl#D ),,["=AABRTXYN!$66~FIIQW_jIk%%a( .q!WfW}#$9Rbc*2J'%%qc*
.*t"3BzB$FN	!z%889
&kAoO!O3J+2
G<<I(%2E*EFB(*=*BBAq"R%K.K  3&$4H
 /7
+?+ fQi$.>M*M')O;?OOH;Xq	+1!9(L M !a%=A%"*K!'AJ%%dY-=&=&AB"'"2"28"<K#&#3x"?*{BZ"ZK!*e](:: "+ a
' ,* #.F1aB;"e] 22 #))1#.Jk:#4C
CA2%Q =T 	
D1M 	
D1s   D=M M2)g      @rU   r   )r)   rU   )gfffff3@g?gMbP?rU   )r6   r?   rU   )r)   r   )r   )NNNr)   r)   N)NNNr  )NNNr6   r)   N)NNNrA  rh  ri  rh  r6   r)   r6   r  r6   r)   NF)NNNr)   r)   Nr?   rf  )NNNr)   r)   Nr  )NNNr)   r)   Nr  )NNNNN)NNNN)NNNNr)   Nr6   )NNNrA  tab)NNNr)   Nr)   F)NNNr)   N)NNNr  F)NNNr  )NNNr)   NNrA  )NNNr)   r)   Nr?   rq  )NNNr  )NNNr)   r)   Nr  )NNNr)   r)   NrT  rg  )
NNFNr)   NrA  r  FF)	NNFNr)   NrA  r  F)NNNr   )\r%   	functoolsr   scipyr   r   r   rh   	tqdm.autor    r   r	   r
   comfy.model_patcherr   comfy.model_samplingcomfy.memory_managementcomfy.utilsr   r   r   r#   r(   r+   r5   rF   rL   rS   r`   rb   r   r   r   r   Tensorr   r   no_gradr   r   r   r   r   r   r   r   r   r  r
  Moduler  rw  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r%  r'  r,  r2  rE  rJ  rL  rO  rQ  rW  rY  ro  rv  rz  r|  r  r  r  r  re   r   r   <module>r     sq                .,*=
 
A!+ !+H: :2$%,, 5<< 
$%,, $5<< $
 .2T4Y[dfotuzo{  FH  2  .  8 -1D$XZcenstynz  EG  D .2T4Y[dfotuzo{  FH    F  >    D
>  . :Q		 Qh [ [   " "J 0 0f 6 6r  2 3 3l | | ; ;| c c A A | | e e  r r  B 04tT[]fhqvw|q}  HJ 5 5t&V=D - -` ) )X ` `
 ) )V  @ > >@ j j i i k k j j ( (V S S ? ?D < <| q q
 ~ ~
 ? ?D e eP Z Z
 NR()p pr   