
    ^(j&                         d dl Z d dlZdZ	 d dlZej
                  ZdZesd ZddZ
ddZddZd	 Zdd
edej                  fdZd ZddZddZddZy# eef$ r  e j                  d       Y Rw xY w)    NFTzTcomfy_kitchen does not support stochastic FP8 rounding, please update comfy_kitchen.c                     t        d      )Nz6comfy_kitchen does not support stochastic FP8 rounding)NotImplementedError)valuerngdtypes      5/Users/danicosta/Desktop/Flux2/ComfyUI/comfy/float.py_ck_stochastic_rounding_fp8r	      s    !"Z[[    c                 ,   t        j                  || d||z
  z  z  dz
  d|z  z  | d| dz   |z
  z  z        }|t        j                  |j                         |j                  |j
                  |j                  |      z  }|j                         d|z  z  S )N       @      ?      r   layoutdevice	generator)torchwhererandsizer   r   r   floor)abs_xexponentnormal_maskMANTISSA_BITSEXPONENT_BIASr   mantissa_scaleds          r   calc_mantissar      s    kk	#(]23	4s	:q-?OP	#=.1,}<=	>O uzz/"6"6"8@U@U^m^t^t  ~M  ~T  ~T  `i  j  jO  "a&677r
   c                    |t         j                  k(  rd\  }}}n%|t         j                  k(  rd\  }}}nt        d      | j	                         } t        j
                  |       }| j                         }t        j                  |dk(  d|      }t        j                  t        j                  t        j                  |            |z   dd|z  dz
        }|dk(   }	t        |||	|||      |d d  |t        j                  |	d||z
  z  d	|z   z  d| dz   z  |z        z  }t        j                  |      }
t        j                  ||
j                  |
j                  |
       |S )N)         )   r      zUnsupported dtyper   r   r   r   r   r   )minmaxout)r   float8_e4m3fnfloat8_e5m2
ValueErrorhalfsignabsr   clampr   log2r   finfor'   r(   )xr   r   EXPONENT_BITSr   r   r.   r   r   r   infs              r   !manual_stochastic_round_to_float8r6      sL   ###6=3}m	%##	#6>3}m,--	A::a=DEEGE;;uz1d+D {{EJJu%&6	1maH M"KUHk=-clmE!HEKK	M)	*sU{;	-!#	$- D ++e
C	KK#''swwD9Kr
   c           	         |t         j                  k(  r | j                  t         j                        S |t         j                  k(  r | j                  t         j                        S |t         j                  k(  r | j                  t         j                        S |t         j
                  k(  s|t         j                  k(  r3t        j                  | j                        }|j                  |       t        rYt        j                  dd| j                         t         j                  | j                  | j                  |      }t        | ||      S t        j                   | |      }t#        d| j%                         dz        }t#        dt'        | j(                  d   |z              }t+        d| j(                  d   |      D ]+  }||||z    j-                  t/        | |||z    ||             - |S | j                  |      S )	N)r   r   r      r   r      r&   )r   float32tofloat16bfloat16r*   r+   	Generatorr   manual_seed!_CK_STOCHASTIC_ROUNDING_AVAILABLErandintr   uint8r   r	   
empty_liker(   numelroundshaperangecopy_r6   )	r   r   seedr   r   output
num_slices
slice_sizeis	            r   stochastic_roundingrO   A   s   xxemmx,,xxemmx,,xxennx--###u0A0A'AOO5<<8	d#,--3

EKKPUP\P\ejeqeq  ~G  HC.uc5AA!!%u5U[[]k:<
E%++a.:"=>?
q%++a.*5A1Qz\"(()J5QRSTU_S_K`bgs|)}~ 688%8  r
   c                    | j                   }t        j                  |       j                  t        j                        }t        j
                  t        j                  | j                               dz         j                  dd      }| t        j                  | j                         | j                  | j                  | j                  |      dz
  d|dz
  z  z  dz  z  } | j                         } t        j
                  t        j                  |       d	z         j                  dd      }t        j                  |dkD  | d|d
z
  z  z  dz
  dz  | dz  |       j                         j                  t        j                        }~ |j                  t        j                        }|dz  |d
z  z  |z  }~~~|j!                  d      }|dd d   dz  |d
d d   z  }|j#                  t%        |      d d dgz         S )Nr   r   r"   r   g      ?r   r   g      ?gGz?r   )r)   r!   )rG   r   signbitr<   rC   r   r1   r/   r0   r   r   r   r   r   r   rF   viewreshapelist)	r3   r   
orig_shaper.   expmantissafp4fp4_flatpackeds	            r   stochastic_float_to_fp4_e2m1r\   Z   s   J==u{{+D
++ejj)C/
0
6
6q!
<C%**QVVXQWWQXXahhZc
dgj
jopux{~u~o  
A  DH  
H  HA	A
++ejjmf,
-
3
3Aq
9C{{a	
ccAg	#	%,	
S	
 egbbo  	

&&
C19
"X
-Cc8xx|Hqt!tn!Xadd^3F>>$z*3B/2$677r
   flattenreturnc                    d }| j                   \  }} ||d      } ||d      }|dz  }|dz  }| }	||f||fk7  r9t        j                  ||f| j                  | j                        }	| |	d|d|f<   |	j                  |d|d      j                  dddd	      }
|
j                  d
ddd      j                  dd      j                  d
dd      }|r|j                         S |j                  ||      S )aY  
    Rearrange a large matrix by breaking it into blocks and applying the rearrangement pattern.
    See:
        https://docs.nvidia.com/cuda/cublas/index.html#d-block-scaling-factors-layout

    Args:
        input_matrix: Input tensor of shape (H, W)
    Returns:
        Rearranged tensor of shape (32*ceil_div(H,128), 16*ceil_div(W,4))
    c                     | |z   dz
  |z  S Nr    )abs     r   ceil_divzto_blocked.<locals>.ceil_div   s    A	ar
      r!   )r   r   Nr   r   r   r"   rQ          )
rG   r   zerosr   r   rS   permuterT   	transposer]   )input_matrixr]   re   rowscolsn_row_blocksn_col_blockspadded_rowspadded_colspaddedblocks
rearrangeds               r   
to_blockedrv   v   s     ##JD$D#&LD!$L $K"KFd|[11+&&&$$

  ,uuete| [[sL!<DDQ1aPFAr1-771=EEb"bQJ!!##k;77r
   c                 :   d}d}| j                   }d}| j                  |d   d|      } t        j                  t        j                  t        j
                  |       d      |z  |j                  | j                        z  |      j                  t        j                        }| |j                  | j                        |j                  | j                        z  j                  d      z  } | j                  |      j                         } t        | |      }||fS )	Ng      @      |@rh   r   rQ   dim)r(   r&   )rG   rT   r   r0   amaxr/   r<   r   r*   	unsqueezerS   
nan_to_numr\   )	r3   per_tensor_scaler   F4_E2M1_MAXF8_E4M3_MAXrV   
block_sizescaled_block_scales_fp8data_lps	            r   %stochastic_round_quantize_nvfp4_blockr      s   KKJJ			*Q-Z0A#kkEJJuyy|,LP[+[_o_r_rstszsz_{*{  BM  N  Q  Q  RW  Re  Re  f	  ),C,F,Fqww,OOZZ[]^^A	z%%'A*1	BG+++r
   c                    dt         dt         dt         fd}t        j                  | j                        }|j	                  |       |r_| j
                  \  }} ||d      } ||d      }	||k7  s|	|k7  r4t        j                  j                  j                  | d|	|z
  d||z
  f      } t        | ||      \  } }
| t        |
d	      fS )
Nr3   multipler^   c                     | |z   dz
  |z  |z  S z#Round up x to the nearest multiple.r   rb   r3   r   s     r   roundupz0stochastic_round_quantize_nvfp4.<locals>.roundup       X!h.(::r
   r8   rh   r   Fr]   )intr   r?   r   r@   rG   nn
functionalpadr   rv   )r3   r~   pad_16xrJ   r   r   rm   rn   rq   rr   blocked_scaleds              r   stochastic_round_quantize_nvfp4r      s    ;3 ;# ;# ; qxx0I$ WW
ddB'dB'$+"5##''A{T/A1kTXFX+YZA=aAQS\]A~j777r
   c                    dt         dt         dt         fd}| j                  }|rk| j                  \  }} ||d      }	 ||d      }
|	|k7  s|
|k7  r@t        j                  j                  j                  | d|
|z
  d|	|z
  f      } | j                  }t        |      }t        j                  |d d |d   dz  gz   t        j                  | j                  	      }t        j                  |d d |d   dz  gz   t        j                  | j                  	      }t        j                  | j                  
      }|j                  |       t        d| j                         |z        }t        dt        | j                  d   |z              }t!        d| j                  d   |      D ]G  }t#        | |||z    ||      \  }}||||z    j%                  |       ||||z    j%                  |       I |t'        |d      fS )Nr3   r   r^   c                     | |z   dz
  |z  |z  S r   rb   r   s     r   r   z9stochastic_round_quantize_nvfp4_by_block.<locals>.roundup   r   r
   rh   r   rQ   r   )r   r   r8   r   r&   Fr   )r   rG   r   r   r   r   rU   emptyrC   r   r*   r?   r@   r(   rE   rF   rH   r   rI   rv   )r3   r~   r   rJ   r   r   rV   rm   rn   rq   rr   
output_fp4output_blockr   rL   rM   rN   rY   blocks                      r   (stochastic_round_quantize_nvfp4_by_blockr      s   ;3 ;# ;# ; J WW
ddB'dB'$+"5##''A{T/A1kTXFX+YZA Jj!JZ_
2!0C/DDEKK`a`h`hiJ;;z#2*R.B2F1GGuObObklkskstLqxx0I$QZ/1JQqwwqzJ679J1aggaj*-:1QJ;OQamvw
U1Q^$**3/Qq:~&,,U3 .
 z,>>>r
   c                    d }|r_| j                   \  }} ||d      } ||d      }||k7  s||k7  r4t        j                  j                  j	                  | d||z
  d||z
  f      } d}d}	d}
| j                   \  }}| j                  |d|
      }t        j                  t        j                  |      d      }t        j                  |j                         |z  d	      }t        j                  t        j                  t        j                  |            j                  t        j                        |	z   dd
      }|j                  t        j                        }|dk(  }|j                  t        j                        dz  j                  t        j                         }t        j"                  |t        j$                  |      |      }|j                         |j'                  d      z  j                  ||      }t)        |t        j*                  |      }t        j"                  |t        j,                  |      |      }|t/        |d      j                  t        j0                        fS )Nc                     | |z   dz
  |z  |z  S ra   rb   )x_valr   s     r   r   z9stochastic_round_quantize_mxfp8_by_block.<locals>.roundup   s    !A%(2h>>r
   rg   r   rx      rQ   ry   g       8)r'         )rJ   Fr   )rG   r   r   r   r   rT   r{   r/   r0   floatceilr1   r<   int32rC   rS   r;   r   	ones_liker|   rO   r*   
zeros_likerv   float8_e8m0fnu)r3   pad_32xrJ   r   rm   rn   rq   rr   r   	E8M0_BIAS
BLOCK_SIZE	x_blockedmax_absscale_needed
exp_biasedblock_scales_e8m0	zero_maskblock_scales_f32data_scaled
output_fp8s                       r   (stochastic_round_quantize_mxfp8_by_blockr      s   ? WW
ddB'dB'$+"5##''A{T/A1kTXFX+YZAKIJJD$		$J/Ijj9-26G ;;w}}<)LLUZZ

<(@ADDU[[QT]]_`befJ"ekk2AI),,U[[9R?EEemmT{{9eoo>N.OQab ??$'7'A'A"'EENNtUYZK$[%2E2EDQJIu/?/?@Q/RTefz"3UCHHI]I]^^^r
   )N)r   )T)r   r:   )loggingr   rA   comfy_kitchenckstochastic_rounding_fp8r	   AttributeErrorImportErrorwarningr   r6   rO   r\   boolTensorrv   r   r   r   r   rb   r
   r   <module>r      s     $) !l"$"<"<(,% )\8 H!288&8d &8ell &8R,"8("?J!_ 	$ lGOOjkls   A A10A1