0jFddlZddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z ddl m Z ddlmZej j#ej j%eZej j)edgZdadadadadad Zej8d ZGd d eZd Z dZ!dZ"dZ#dZ$dZ%GddeZ&Gdde Z'y)N)Path)knobs) GPUTarget) GPUDriver) _allocation)compile_module_from_srcincludec ddl}|jdk7ryddl ddlm}m}m}m m m}G fdd j} j|||||||} jdj}| g|_||_d j!dz} fd } ||| | r$t#j$ j'| Sy#t$rYywxYw) NrLinux)c_charc_intc_size_tc_void_pc_char_pPOINTERc"eZdZdWfdWfgZy)8_find_already_mmapped_dylib_on_linux..DlPhdrInfo dlpi_addr dlpi_nameN)__name__ __module__ __qualname___fields_)rrsY/var/www/html/ia-enhanced/venv/lib/python3.12/site-packages/triton/backends/amd/driver.py DlPhdrInfor!s ( # ( # rz libc.so.6ic |jj}ttj|}|j vr'j ||tt|yy)Nrr) contentsrrosfsdecodenamememmoveminlen)infosizedatarpctypeslib_namemax_path_lengths rcallbackz6_find_already_mmapped_dylib_on_linux..callback8sRMM++ Y' ( qvv  NN4CY,P Qr)platformsystemr*r r rrrr Structure CFUNCTYPECDLLdl_iterate_phdr Exceptionargtypesrestypecreate_string_bufferr r! string_at)r+r.r r rrr callback_tr3pathr-rrr*r,s` @@@@r$_find_already_mmapped_dylib_on_linuxr;sG# KK V%% !!%)OQXY_Q`aJ ++k2BB!+H5O#OO  & &': ;Dz(+T2{{6++D122 + s1C-- C98C9c d}tjjx}rC|j|r!tj j |r|Std|d|t|}|r2tj j |r|Std|d|g}tj jtj jtd|}tj j |r|S|j|ddl }|j}|j}|j r|g|z}|D]X}tj j|dd|}tj j |r|cS|j|Zt j"d } | rj| j%d D]V} tj j| |} tj j | r| cS|j| Xt j"d } | rStj j| d|} tj j | r| S|j|  t'j(d d gj+j-}|rStj j|d|} tj j | r| S|j| t j"d}|rStj j|d|}tj j |r|S|j|t'j(ddgj+d}|j3Dcgc]5}|j-j|s#|j%d7}}|D]6}tj j |r|cS|j|8tj jd|}tj j |r|S|j|td|d|#t&j.t0f$rYwxYwcc}w)Nzlibamdhip64.sozTRITON_LIBHIP_PATH 'z' does not point to a valid zmemory mapped 'z'' in process does not point to a valid librtorchLD_LIBRARY_PATH:HIP_PATH hipconfigz--path ROCM_PATHz/sbin/ldconfigz-pignore)errorsz/opt/rocm/lib/zcannot locate z after attempted paths )ramd libhip_pathendswithr r:exists RuntimeErrorr;joindirname__file__appendsitegetsitepackagesgetusersitepackagesENABLE_USER_SITEgetenvsplit subprocess check_outputdecodestripCalledProcessErrorFileNotFoundError splitlines)r+env_libhip_path mmapped_pathpaths local_librP site_packages user_siter:env_ld_library_pathdf env_hip_path hip_lib_pathhip_root env_rocm_path rocm_lib_pathlibslinelocsloccommon_install_paths r_get_path_to_hip_runtime_dylibrpFsH ))////  # #H -"''..2Q" "1/1BB^_g^hijj8AL 77>>, ' _\N:abjaklmm E RWW__X6xHI ww~~i  LL((*M((*I " m3 ww||D'5(; 77>>$ K T ))$56$**3/ A Q)Aww~~a  LLO  99Z(Lww||L%B 77>>, '  \"  **K+BCJJLRRT 77<<%BLww~~l+## LL & IIk*M ]E8D 77>>- (  ]#  " "$4d#; < C C8 C TD*.): ^djjl>S>ST\>]DJJL  ^D ^ 77>># J S '',,'7B ww~~)*"" LL$% z1HP QQ;  ) )+< =   " _s%A6R R$R;&R;R87R8c$eZdZfdZdZxZS)HIPUtilscdt|dstt|||_|jS)Ninstance)hasattrsuperrr__new__rt)cls __class__s rrwzHIPUtils.__new__s*sJ' 37%>"%(%>%>"jj (+(D(D% --oo )) ^^ MM r)rrrrwr __classcell__rys@rrrrrs  "rrrc\|jdry|dk(rydddddd d d d d d d d d d d|S)N*hipDeviceptr_t tensordesc TDMDescriptorint8_tint16_tint32_tint64_tuint8_tuint16_tuint32_tuint64_tdouble)i1i8i16i32i64u1u8u16u32u64fp16bf16fp32f32fp64) startswith)tys r ty_to_cpprs] }}S \   !  rcg}d}|D]C}t|tr|jdr |r||nd}|dz }tjd|}|j d}|j d}|j ddz} |[|jd|ztd| zD]} |jd|jd |jd n|jdt| D]} |jd t| D]} |jd3|j|F|S) Nrrrztensordesc<([^[>]*)\[([^]]*)\],rrrr) isinstancestrrrematchgroupcountrOrange) signaturetensordesc_metaoutputtensordesc_idxsigmetardtypeshapendim_s rexpand_signaturers7 FN c3 CNN<$@6E?>24D a NHH?EEKKNEKKNE;;s#a'D| cEk*q4x)AMM%() d# d# l+4[ % e$ %4[ % e$ % MM# 36 Mrcfdg}|D] }|| |Dcgc] }|dk7s | }}tjjjjj |Scc}w)zj Creates a kernel signature in C to be able to efficiently extract arguments in the launcher. cjt|tr|D] }|| y|j|yN)rtuplerO)rrx_flatten_signatures rrz1make_kernel_signature.._flatten_signatures4 c5 ! ."1f- . MM# r constexpr)tritonruntimedriveractiveutilsr)rflat_signaturerrkernel_signaturers @rmake_kernel_signaturersq N03/0#1FaQ+5EFF >> ' ' - - F FGW XXGs A+A+cg}|D]}t|tr*|jtt |t =|dk7r!|jtdt c|jtdt|S)z This recreates the signature with annotations as C objects which can then be used to efficiently flatten tuples, and remove constexpr in the launcher. ) nested_tupletyperN)rrrOrannotate_argumentsrrr)rannotated_argumentsrs rrrs| [ c5 !  & & ASTWAX_h(i k K   & &{:'V W  & &{='Y Z [ rc |X|jg|j|j|jdk(|j|j|jS|j}|j}|jj }d|vrd|vsJ|d}|d}d\}} |j dg} | r)t| dk(rt| ddk(sJ| d\}} |d} tjjj} t| tsJ| jj||| || |||} | g||S) a Translate a tensor descriptor argument into the appropriate list of kernel arguments. If `tensordesc_metadata` is provided, we will create a TDMDescriptor object. Otherwise, we decompose the tensor descriptor into base pointer, shape, strides, and padding flag. In both cases, we append the shape and strides at the end to match the expected kernel signature. nan elem_bits block_size)rrinterval_padding_pairsrrr)baserstridespaddinground_f32_to_tf32data_ptrgetr%rrrrr HIPDriverrr)argkernel_metadatatensordesc_metadatarrrrr pad_interval pad_amountr num_warpsrdescs rmake_tensordesc_argr&s" HH YY [[  KK5   ! !  YY  [[   IIEkkG 88   D - -,BU2UU U#K0I$\2J#L*0445MrR)*a/C8Nq8Q4RVW4WWW#9!#< j"I ^^ " " ) )F fi (( ( << - -iYP\^hjo.5t =D  #5 #7 ##rc vtd|jD}|sStt|jDcgc]*\}}t |t s|j ds)|,c}}rttk(sJsdgtzfd}|Scc}}w)a Wrap a kernel launcher function to handle tensor descriptor arguments. Use the provided `tensordesc_metadata` to determine whether to create TDMDescriptor objects or decompose the tensor descriptors. Args: launcher (callable): The original kernel launcher function. signature (Dict[int, str]): The kernel signature mapping argument indices to types. tensordesc_metadata (List[Dict] or None): The list of tensor descriptor metadata, following the order of tensor descriptor arguments. If None, decompose tensor descriptors. Returns: launcher (callable): The wrapped kernel launcher function. c3bK|]'}t|txr|jd)yw)rN)rrr).0rs r z)wrap_handle_tensordesc..fs)rX[jc2Ss~~l7SSrs-/rNc |dd}|d}|d}g}d}t|D]?\}}| vr%|jt|| ||dz }/|j|Ag||S)NrFrr) enumerateextendrrO) args base_argsr kernel_argsfinal_kernel_argsrirlaunchertensordesc_indicesrs rinnerz%wrap_handle_tensordesc..innerps"I #A,2h  , .FAs&&!(()#FXBY&YY Y #fs+='>>7 L+ qs B5 "B5 4B5 ceZdZdZdZy) HIPLauncherctdr jn t}fd}|jDcic]\}}|||}}}jjDcic]\}}|| }}}t |dd}t jjjjj} t|j|} t| |_t!| |_t%| |||_ |j&|_|j(|_|j&rmt jjj} t+| t,sJ| j/} | jj1| } | dsJd|j2|_|j4|_ycc}}wcc}}w)N constantsctt|tr&jjj |fS|Sr)rrfn arg_namesindex)rr|s rz&HIPLauncher.__init__..s-Z3=OSVV--33A69UVrrcooperativeLaunchz8Cooperative launch requested but not supported by device)rurdictitemsrgetattrrrrrrrrrrarg_annotationsrrrlaunch_cooperative_grid warp_sizerrget_current_devicerprofile_scratch_sizeprofile_scratch_align)rr|metadatararg_idxidxvaluerrrexpanded_signaturerdevicedevice_propertiess ` rrzHIPLauncher.__init__s%,S+%>CMMDF V;D??;LMZS%WS\5(M M25--2E2E2GHJCS%ZH H!(,=tD>>((//55<<-i.>.>.@/R12DE 56H I,Xy/R '/'G'G$!++  ' '^^**11Ffi0 00..0F & B B6 J $%89 KJ K9$,$A$A!%-%C%C"'NHs G/ Gc fd} | |j|jtj} |j |j || |||| |j |j|j| y)Nc^|dkDr'zz}||z}|j}||| SyNr)r) r'align allocator grid_size alloc_sizealloc_fngridXgridYgridZstreams rallocate_scratchz.HIPLauncher.__call__..allocate_scratchs?ax!EME1 &- $==? E6::r) rrr_profile_allocatorrrrrr) rr#r$r%r&functionrlaunch_metadatalaunch_enter_hooklaunch_exit_hookrr'profile_scratchs ```` r__call__zHIPLauncher.__call__sy +4+D+DdF`F`+6+I+IK D00%vxYh#_6GIY[_[i[i(($*?*? GrN)rrrrr.rrrrsD0GrrcbeZdZfdZdZedZdedefdZdZ dZ d Z d Z d Z xZS) rcVt|t|_t|_yr)rvrrrrr launcher_cls)rrys rrzHIPDriver.__init__s Z 'rc"ddl}|jSr)r>cudarr>s rget_device_interfacezHIPDriver.get_device_interfaceszzrc ddl}|jjxr|jjduS#t $rYywxYw)NrF)r>r4 is_availableversionhip ImportError)r>s r is_activezHIPDriver.is_activesC  ::**,P%--2C2C42O P  s7: AArreturnct|Sr)r)rrs rmap_python_to_cpp_typez HIPDriver.map_python_to_cpp_types }rc|j}|jj|}tjj xs|d}|d}t d|jdd|S)NarchwarpSizer:r@r)rrrrr override_archrrU)rrrrArs rget_current_targetzHIPDriver.get_current_targetse((* JJ<>rcJddl}|jd|jS)Nrr4)r>rrr5s rget_active_torch_devicez!HIPDriver.get_active_torch_devices||FD$;$;$=>>rcddlm}|S)Nr)do_bench)triton.testingrH)rrHs rget_benchmarkerzHIPDriver.get_benchmarkers +rcbddl}d}|jt|dz|jdS)Nrir4)rr)r>emptyint)rr> cache_sizes rget_empty_cache_for_benchmarkz'HIPDriver.get_empty_cache_for_benchmarks.' {{3zQ/uyy{PPrc$|jyr)zero_)rcaches r clear_cachezHIPDriver.clear_caches  r)rrrrr6 staticmethodr<rr?rDrFrJrPrTrrs@rrrsN( ?? Qrr)( functoolsr rVrrpathlibrrtriton.backends.compilerrtriton.backends.driverrtriton.runtimertriton.runtime.buildrr:rMrealpathrNrLr}rrrrrr; lru_cacherpobjectrrrrrrrrrrr/rrr_s  .,&8 ''//"''**84 5 Wi01     -`\R\R~"v"B 0BY,  .$b)X*G&*GZ. .r