SjR UdZddlmZddlZddlZddlmZerddlmZddl m Z ddl m Z ddl mZmZddlZdd lmZd d lmZd gZdad ed<dad ed<e GddZe d.dZ d/ d0dZd1dZd2dZ d3dZ d4dZ d5dZ edZ!d6dZ"d7dZ# d8 d9dZ$ d: d;dZ% dd#Z'ddddddddd$ d?d%Z(dddd& d@d'Z) dAdd( dBd)Z* dCdd( dDd*Z+dd( dEd+Z,ejZd,e-y)Fz PROTOTYPE! Flash Attention 3 implementation. For fp8: only supports forward pass right now. For fp16/bf16: supports forward and backward pass. ) annotationsN) TYPE_CHECKING)Callable) dataclass)cache) TypeVarTupleUnpack)Library) _registryregister_flash_attention_fa3zCallable | None _FA3_CUDA_FWD _FA3_CUDA_BWDc eZdZUded<ddZy) _FA3HandlezLibrary | NonelibrarycPd|_tjjdy)NF)rtorch_C_set_sdp_use_fa3)selfs V/var/www/html/ia-enhanced/venv/lib/python3.12/site-packages/torch/nn/attention/_fa3.pyremovez_FA3Handle.remove*s  !!%(N)returnNone)__name__ __module__ __qualname____annotations__rrrrr&s )rrcJtjj|\}}|SN)rcudaget_device_capability)devicemajor_s r_get_device_majorr)0szz//7HE1 Lrc|t|tjjdt t S)z Register FA3 flash attention kernels with the PyTorch dispatcher. Args: module_path: Python module path to the FA3 implementation. T)_fa3_import_modulerrrr_fa3_register_kernels module_paths rr r 6s/{# HHd# +- ..rctj|ttjdst d|dttjj dst d|dttjj dst d|dtjj jatjj ja y)N flash_attn_3zModule 'z' does not expose FA3 kernelsfwdz%' does not expose FA3 forward kernelsbwdz&' does not expose FA3 backward kernels) importlib import_modulehasattrrops RuntimeErrorr0r1rr2rr-s rr+r+Gs K( 599n -Xk]2OPQQ 599))5 1{m#H I   599))5 1{m#I J  II**..MII**..Mrcbtddd}|jdtd|jdtd|jdtd|jdt d|jdt d|jd td|jd td|S) NatenIMPLCUDAz"_flash_attention_forward.quantizedz-_scaled_dot_product_flash_attention.quantized_flash_attention_forward+_flash_attention_forward_no_dropout_inplace#_scaled_dot_product_flash_attention_flash_attention_backward,_scaled_dot_product_flash_attention_backward) r impl!_fa3_flash_attention_forward_impl4_fa3_scaled_dot_product_flash_attention_forward_impl)_fa3_flash_attention_forward_impl_default4_fa3_flash_attention_forward_no_dropout_inplace_impl<_fa3_scaled_dot_product_flash_attention_forward_impl_default"_fa3_flash_attention_backward_impl5_fa3_scaled_dot_product_flash_attention_backward_impl)libs rr,r,Xs &&& )CHH,.OQWHH7< HH"$MvHH5< HH-D HH (*LfUHH6= Jrc|dk7rytd|Dsyt|Dchc]}|jc}dk7ry|jtj k(r |||t jdt||jdk7ry ||jd k7ry tjjsy t|jd k7ryycc}w)Nzdropout_p must be 0c34K|]}|jywr#)is_cuda.0ts r z,_fa3_common_support_error..s*Qqyy*szinputs must be CUDA tensorsr inputs must share devicezWhen using SDPA with fp8, descale tensor should always be used for accurate dequantization. Please use _scaled_dot_product_attention_quantized and provide the descale tensors.zdense query must be 4Dzragged query must be 3DzCUDA not available z#FA3 requires compute capability 9.0) alllenr&dtyper float8_e4m3fnwarningswarn UserWarningdimr$ is_availabler))querytensors dropout_p cum_seq_q q_descale k_descale v_descalerPs r_fa3_common_support_errorrfysC$ *'* *, g &AHH &'1,) {{e)))Y.)2C  +   UYY[A-'!1( :: " " $#&!+4 ) 'sC#c  |ry|y|+|jtjk7ry|jsytjtj tj f t fd|||hDsd St|||hD chc]} | jc} dk7ryt||||f|||| | } | | d k(ry | Sycc} w) Nzreturn_debug_mask must be Falsezalibi_slopes not supportedzseqused_k must be int32zseqused_k must be CUDAc3:K|]}|jvywr#rXrOrPsupported_dtypess rrQz-_fa3_forward_support_error..sHqqww**Hinputs must be one of r #all inputs must have the same dtyperRz(query, key, value must be on same device) rXrint32rMrYfloat16bfloat16rVrWrf)r_keyvaluerareturn_debug_mask alibi_slopes seqused_krbrcrdrerPerrorrks @r_fa3_forward_support_errorrxs0+ ??ekk ),  +++U]]ENNK HUC4GH H'(8'9:: eS%0 1AGG 12a74 % U E  . .=  2s C c  |jtjk(r y|jtjk7rytjtj f t fd|||||hDsd St|||||hD chc]} | jc} dk7ryt|||||||f||ddd} | | Sycc} w)NzHFA3 backward does not support fp8 - use inference only (torch.no_grad())zlogsumexp dtype must be float32c3:K|]}|jvywr#rirjs rrQz._fa3_backward_support_error..sWqqww**Wrlrmr rn) rXrrYfloat32rprqrVrWrf) grad_outr_rrrsout logsumexprarbwindow_size_leftwindow_size_rightrPrwrks @r_fa3_backward_support_errorrs {{e))) V %--'0 u~~6 WXuc5RU4VW W'(8'9:: hsE3? @AGG @AQF4 % 5#uc95    E   As CTsc&td|DS)Nc3@K|]}|jddyw)r N) transposerNs rrQz#_transpose_dense..s4qQ"4s)tuple)r`s r_transpose_densers 4G4 44rcR|$|jddk7r|jS|S)z2Ensure tensor is contiguous in the last dimension.r )stride contiguous)xs r_maybe_contiguousrs&]qxx|q/@1<<>GaGrct tdt|}t|}|jtj k(r8|j ddk7r$|j ddk7r|jn t|}t|}t|}t| } t|}tg|||ddd| ||dd| |||ddddd| ||||| | nd| | nddddd|xst jrdnddtjjxsd\} }}}| |jfS) zF Run the FA3 forward pass by calling the C++ kernel directly. NFA3 not registeredrr rrKT) rr7rrXrrYrr$are_deterministic_algorithms_enabledr_get_sm_carveout_experimental)r_rrrscu_seq_qcu_seq_kmax_qmax_kscale is_causalrrrvr}rcrdre block_table num_splitsqkv cu_seqlens_q cu_seqlens_k softmax_lse out_accumsoftmax_lse_accums r_fa3_run_forwardrs`./00% A#A ;;%-- - LL  ! LL  ! u % %X.L$X.L!),I#K0K5B$6 $6 $6 $6 $6 $6 $6 $6 $6 $6 $6 $6 $6 $6 $6 $6 !$6" #$6$ %$6& '$6( )$6* +$6, -$6. /$60 1$62 3$64-8b5$66/:7$68 9$6: ;$6< =$6> ?$6@  F;;=A1C$6D E$6F ..05AG$62Ci!2J  &&( ((rcRt tdt|}|jddk7r|j n|}|jddk7r|j n|}|jddk7r|j n|}t|}t|}t j |}t j |}t j |}t|||||||||||dd|| | | | | d|t jjxsd|||fS)Nrrr rKr) rr7rrrr empty_likerr)r|r_rrrsr}r~rr max_seqlen_q max_seqlen_krrrr deterministicdoutrrrolsedqdkdvs r_fa3_run_backwardrLs/"/00 X &D#ll2.!3AJJrNa/SA#ll2.!3A#A I &C   ! B   ! B   ! B              ..05A-0 r2:rrT rrrrvrur}rcompute_auxiliaryrc t||||| |||| | | }|td|t|||||||| |||||| | | ||\}}|rtjdtj |j }tjdtj |j }tjd|j|j }nd}d}d}|||||fS)Nz)FA3 flash_attention forward unsupported: )r)rXr&r!r) rxr7rrzerosuint64r&emptyrX)r_rrrsrb cum_seq_krrrarrtrcrdrerrrrvrur}rrrrwr rng_state philox_offset debug_masks rrBrBs2 '    E FugNOO       %HC(KKELLN  Bell5<<P [[%++ellK    Y z 99r)rrrrvrurrc Rt||||||||| | dddf | | | ||||d|d \}}}}}|S)NFrrB)r}r_rrrsrbrrrrarrtrrrrvrurrr(rs rrErEsh*8        )+! -OAsAq!0 Jr)rrrrvrurr}rc Bt|||||||||| ddd| | | | ||||S)N)rrrrvrur}rrr)r_rrrsrbrrrrarrtrrrrvrurr}rs rrDrDsP* -        )+! + r)rrrct||||||| ||| }|td|tj}t |||||||||| || ||nd||nd|\}}}|||fS)z0FA3 implementation of _flash_attention_backward.z*FA3 flash_attention backward unsupported: r)rr7rrr)r|r_rrrsr}r~rbrrrrarrunusedrrrrwrrrrs rrGrG,s* (     E GwOPP>>@M"       ,8b.:JBB" r2:rrc t|||||ddd||| } | td| t|||\} } } |jtj k(rtj n |j}t j||}|jdd}| jd}| jd}t| | | dd|||||| ||||\}}}}}|jd}|jd}||dd|||||f S)NzFA3 SDPA forward unsupported: rir r)rr}rcrdre) rxr7rrXrrYrqrrsizerB)r_rrrsrcrdrerarrtrrwrrr out_dtypeout_bhsdout_bshd max_q_flash max_k_flashr(rrrrrrs rrCrCgsA '       E ;E7CDDuc51GAq! #(++1D1D"D%++IY7H!!!Q'H&&)K&&)K3T      40AsI}j" JJqME HHQKE       rc ,t|||ddd|||| S)Nr)rC)r_rrrsrarrtrs rrFrFs0 @        rct||||||| ddd }|td|t|||||\}}}}}t||||||dd|| | | | | |\}}}t|||\}}}|||fS)zCFA3 implementation of _scaled_dot_product_flash_attention_backward.NzFA3 SDPA backward unsupported: r)rr7rrG)r|r_rrrsr}r~rbrrrrar philox_seedrrrw grad_out_tq_tk_tv_tout_trrrdq_outdk_outdv_outs rrHrHs& (%eS)YdD E rs# (!2 !#  "& %!% % )) ) .///"/"B" " %""# " # " # "#""J( ( ( ( (  ( & (#(#(#(#(#((V## # #  #  #  ####!#"##L$5H$ $%)%)%)'+!%N) N) N) N)" N) " N)  N) N) N)N)!N)"N)#N) N)#N)#N) #!N)"%#N)$%N)&''N)@ 88 8 8  8  8  8"8"888 88888 5!8L&*%)%)D:%)(,#'+"!/D: D: D: D:# D: # D:  D: D:D:D:D:#D:#D:#D: D: !D:"#D:$#%D:&&'D:( )D:*%+D:,-D:./D:h%)(,'+!'- - - -  - # - # - - ---- --- #!-"&#-$%%-&'-x%)(,'+#!'+ + + +# + # +  + ++++ +++#+ &!+"%#+$ %+&'+|#'$(%88 8 8  8  8  8#8#8 8 8888 8 !8"!#8$"%8~&*%)%)#DD D D D# D # D # DDDD DV#          P!2"2" 2" 2"  2"  2"  2"#2"#2" 2" 2"2"2"2" 2" !2"j( '';WXr