SjA9 UdZddlmZddlZddlmZddlmZddlm Z m Z ddl m Z m Z dd lmZe rdd lmZddlZdd lmZd gZdad ed<eGddZed)dZ d* d+dZed,dZd-dZ d. d/dZ d0 d1dZ d2dZd3dZ e dZ!d4dZ" d5 d6dZ# d7 d8dZ$dddddddddd d9dZ%dddddddd  d:d!Z&dddd" d;d#Z' dd&Z)ejTd'e(y)?zUBER PROTOTYPE!!!) annotationsN) dataclass)cache)Any TYPE_CHECKING) TypeVarTupleUnpack) _registry) ModuleType)Libraryregister_flash_attention_fa4 str | None_FA4_MODULE_PATHc eZdZUded<ddZy) _FA4HandlezLibrary | Nonelibrarycd|_yN)r)selfs V/var/www/html/ia-enhanced/venv/lib/python3.12/site-packages/torch/nn/attention/_fa4.pyremovez_FA4Handle.remove"s  N)returnNone)__name__ __module__ __qualname____annotations__rrrrrs rrcJtjj|\}}|Sr)torchcudaget_device_capability)devicemajor_s r_get_device_majorr(&szz//7HE1 LrcBt|}|attS)z Register FA4 flash attention kernels with the PyTorch dispatcher. Args: module_path: Python module path to the FA4 implementation. )_fa4_import_modulerr_fa4_register_kernels) module_pathr's rrr,s# ;'A" +- ..rc~tj|}t|dr t|dstd|d|S)N_flash_attn_fwd_flash_attn_bwdzModule 'z' does not expose FA4 kernels) importlib import_modulehasattr RuntimeError)r,modules rr*r*;sA  $ $[ 1F 6, -WVEV5WXk]2OPQQ Mrctddd}|jdtd|jdtd|jdtd|jdt d|jdt d|S) NatenIMPLCUDA_flash_attention_forward+_flash_attention_forward_no_dropout_inplace_flash_attention_backward#_scaled_dot_product_flash_attention,_scaled_dot_product_flash_attention_backward)r impl!_fa4_flash_attention_forward_impl4_fa4_flash_attention_forward_no_dropout_inplace_impl"_fa4_flash_attention_backward_impl4_fa4_scaled_dot_product_flash_attention_forward_impl5_fa4_scaled_dot_product_flash_attention_backward_impl)libs rr+r+Cs~ &&& )CHH ')JFSHH5< HH (*LfUHH-< HH6= Jrctd|Dsyt|Dchc]}|jc}dk7ry|jtj tj fvry|D])\}}|jtjk7s$|dcS||jdk7ry||jd k7ry tjjsy t|jd vry ycc}w)Nc34K|]}|jywr)is_cuda.0ts r z,_fa4_common_support_error.._s*Qqyy*szinputs must be CUDA tensorsr inputs must share devicez'query dtype must be float16 or bfloat16z dtype must be float32zdense query must be 4Dzragged query must be 3DzCUDA not available) z+FA4 requires compute capability 9.0 or 10.0) alllenr%dtyper"float16bfloat16float32dimr# is_availabler()querytensors cum_seq_q require_fp32rJnametensors r_fa4_common_support_errorr_Ys *'* *, g &AHH &'1,) {{5==%..998$3 f <<5== (V12 23UYY[A-'!1( :: " " $#&g5<  'sC:c |dk7ry|ry|y|+|jtjk7ry|jsyt |j } | | dk7rd| d S| | d kDr | dk7rd | d St ||||f|} | | d k(ry | Sy)Ndropout_p must be 0zreturn_debug_mask must be Falsezalibi_slopes not supportedzseqused_k must be int32zseqused_k must be CUDArPz+paged KV (block_table) not supported on SM 0r z-SplitKV (num_splits > 1) not supported on SM rLz(query, key, value must be on same device)rSr"int32rGr(r%r_) rYkeyvalue dropout_preturn_debug_mask alibi_slopes seqused_kr[ block_table num_splitsr&errors r_fa4_forward_support_errorrnssC$0+ ??ekk ),  + ell +E5B;ugQGG % U E   . .= rc H|dk7ryt|||||||f|d|ff}||Sy)Nrarb logsumexp)r\)r_) grad_outrYrerfoutrprgr[rms r_fa4_backward_support_errorrssJC$ % 5#uc95"I.0  E   rc|dk(rdS|S)z"need to convert -1 to None for FA4Nr )vals r_aten_to_fa4_window_sizerws"94%#%rTsc&td|DS)Nc3@K|]}|jddyw)r N) transposerHs rrKz#_transpose_dense..s4qQ"4s)tuple)rZs r_transpose_denser~s 4G4 44rct tdtt}||t| t| d||||| | j nd| |xsd| d }|j |||fi|\} }| |j fS)NFA4 not registeredTr ) softmax_scalecausalwindow_size_leftwindow_size_right return_lse cu_seqlens_q cu_seqlens_k max_seqlen_q max_seqlen_krj page_tablerlrr)rr3r*rw contiguousr.)rYrerfcu_seq_qcu_seq_kmax_qmax_kscale is_causalrrrjrrrkrlr4kwargslses r_fa4_run_forwardrs"/00  0 1F45EF56GH  /8/DY))+$! oAF&v%%eS%B6BHC   rc t tdtt} | j||||||j || t | t | |||  \}}}|||fS)Nr)rrrrrr deterministic)rr3r*r/rrw)rqrYrerfrrrprrrrrrrr4dqdkdvs r_fa4_run_backwardrs/00  0 1F''    12BC23DE#(JBB r2:rT rrrrjrirrrkcompute_auxiliaryrlc t||||| || ||| }|td|t|||||||| || | | |||\}}|rtjdtj |j }tjdtj |j }tjd|j|j }nd}d}d}|||||fS)Nz)FA4 flash_attention forward unsupported: )r{)rSr%r r) rnr3rr"zerosuint64r%emptyrS)rYrerfr[ cum_seq_krrrgrrhrrrrjrirrrkrrlrmr rng_state philox_offset debug_masks rr?r? s, '    E FugNOO       HC"KKELLN  Bell5<<P [[%++ellK    Y z 99r)rrrrjrirkrlc Nt||||||||| | | | | ||||d|\}}}}}|S)NFr)r?)rrrYrerfr[rrrrgrrhrrrrjrirkrlr'rs rr@r@JsX*8     )+! 'OAsAq!* Jr)rrrct||||||| |}|td|tj}t |||||||||| ||| \}}}|||fS)Nz*FA4 flash_attention backward unsupported: )rsr3r"$are_deterministic_algorithms_enabledr)rqrYrerfrrrpr[rrrrgrrunusedrrrrmrrrrs rrArAws( (     E GwOPP>>@M"     JBB r2:rrct|||||ddd}|td|t|||\}} } tj|} | j dd} |j d} | j d}t|| | dd| ||||||  \}}}}}|j d}|j d}| |dd|||||f S)NzFA4 SDPA forward unsupported: r r{)rrr)rnr3r~r" empty_liker|sizer?)rYrerfrgrrhrrmqkvout_bhsdout_bshd max_q_flash max_k_flashr'rrrrrrs rrBrBs  '       E ;E7CDDuc51GAq! &H!!!Q'H&&)K&&)K3T       40AsI}j JJqME HHQKE       rct||||||| d}|td|t|||||\}}}}}|jd}|jd} t ||||||dd|| | | | | |\}}}t|||\}}}|||fS)NzFA4 SDPA backward unsupported: r{r)rsr3r~rrA)rqrYrerfrrrpr[rrrrgr philox_seedrrrmrrrogorrrs rrCrCs$ (      E rsx#!%2 !#  $*#   3 / / /4:<   %#7   F(,!% % % % %  % & %#%#%%%%%P        #.& $5" $'+!%! %! %! %!" %! " %!  %! %! %!%!!%!"%!#%! %!%%!%! '!%!j !! ! !  !  !  !"!"! !!!!"!!5!`#'$(%)(,#'+"!)=: =: =: =:# =: # =:  =: =:=:=:=: =:!=:"=:#=: &!=:" #=:$%%=:&'=:()=:Z#'$(%)(,'+!'* * * *  * # * # * * **** *!*"* #!*"&#*$%%*&'*z#'$(%00 0 0  0  0  0#0#0 0 0000 0 !0"!#0$"%0n# :: : : : :  :  : :Z!33 3 3  3  3  3#3#3 3 3333 3 !3l( '';WXr