Ë
    L²j_á ã                  ó4  — U d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlmZmZ d dlmZmZ d dlmZmZmZmZmZ d dlmZ d dlmZ dd	lmZm Z  erd d
l!m"Z"m#Z#m$Z$ d dl%m&Z& d dl'm(Z( ddl)m*Z* d dl+Z+d dl,Z,d dl-Z,d dl.m/c m0Z1 d dl2m3Z3m4Z4 d dl5m6Z6 d dl7m8Z8m9Z9 d dl:m;Z; d dl<m=Z=m>Z> d dl?m@Z@ d dlAmBZB d dlCmDZDmEZEmFZF d dlGmHZH ddlImJZJmKZKmLZLmMZMmZmNZN ddlOmPZP ddlQmRZRmSZSmTZT ddlUmVZVmWZW ddlMmXZXmYZYmZZZm[Z[ ddl\m]Z]m^Z^ ddl_m`Z` ddlmaZambZbmcZcmdZdmeZemfZf ddlgmhZh dd limjZjmkZk dd!llmmZmmnZn dd"lompZpmqZq dd#lrmsZs dd$l/mtZtmuZumvZvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZm€Z€m�Z�m‚Z‚mƒZƒm„Z„m…Z… dd%l†m‡Z‡  e�j                  e‰«      ZŠe,�j                  �j                  e‰d&«      Z�e,�j                  �j                  e‰d'«      ZŽe,�j                  �j                  e‰d(«      Z�e,�j                  �j                  e‰d)«      Z�e‘d*   Z’d+e“d,<    ed-«      Z” ed.«      Z•e�j,                   G d/„ d0«      «       Z—e�j,                   G d1„ d2«      «       Z˜ G d3„ d4«      Z™e�j,                   G d5„ d6«      «       Zše�j,                   G d7„ d8eš«      «       Z› G d9„ d*«      Zœe�j:                  dld:„«       Zždmd;„ZŸdnd<„Z dod=„Z¡ e�j,                  d>¬?«       G d@„ dA«      «       Z¢dpdB„Z£ G dC„ dD«      Z¤	 	 	 	 	 	 	 	 dqdE„Z¥ G dF„ dGeœ«      Z¦ G dH„ dIeœ«      Z§ G dJ„ dKeœ«      Z¨	 	 	 	 drdL„Z©	 	 	 	 	 	 	 	 dsdN„Zª G dO„ dPeœ«      Z« G dQ„ dRe««      Z¬ G dS„ dTe««      Z­ G dU„ dVe««      Z® G dW„ dXeœ«      Z¯	 dt	 	 	 	 	 	 	 dudY„Z°	 	 	 	 	 	 dvdZ„Z±dwd[„Z²	 	 	 	 	 	 	 	 	 	 	 	 dxd\„Z³	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dyd]„Z´e�j,                   G d^„ d_«      «       Zµ e�jl                  «       Z·dzd`„Z¸d{da„Z¹	 	 	 	 d|db„Zºd}dc„Z»d}dd„Z¼d~de„Z½d~df„Z¾d~dg„Z¿d~dh„ZÀ G di„ dM«      ZÁ G dj„ dk«      ZÂy)é    )ÚannotationsN)ÚCounterÚdefaultdict)Úas_completedÚFuture)ÚAnyÚGenericÚTYPE_CHECKINGÚ	TypeAliasÚTypeVar)Ú	ParamSpec©Ú
OrderedSeté   )ÚComputedBufferÚ	Pointwise)ÚCallableÚIteratorÚSequence)Ú
ModuleType)ÚEnterCudaStreamContextLine)ÚPythonWrapperCodegen)ÚcountersÚdynamo_timed)Úuse_pipelined_autotuning)ÚLambdaFutureÚPyCodeCache)ÚTritonTemplateCallerBase)Úget_metric_tableÚis_metric_table_enabled)Úget_stream_name)Úfree_symbols)Úfree_symbol_is_typeÚsymbol_is_typeÚSymT)Ú
has_triton)ÚcommsÚconfigÚconfig_commsÚdependenciesÚirÚmetrics)Úcan_codegen_without_upcasts)ÚBackendFeatureÚget_scheduling_for_deviceÚKernel)Ú estimate_nccl_collective_runtimeÚ/estimate_nccl_collective_runtime_nccl_estimator)ÚDepÚ	MemoryDepÚStarDepÚWeakDep)ÚGPUTooOldForTritonÚTritonMissing)Úcount_flops_fx)Úassign_origin_nodeÚget_device_typeÚGraphPartitionSignatureÚMultiOutputÚMultiOutputLayoutÚ
NoneLayout)ÚLoopBody)ÚMemoryPlanningInfoForBufferÚMemoryPlanningInfoForNode)ÚDevicePropertiesÚReductionHint)Ú
green_textÚred_text)ÚSimplifyIndexing)Ú&_unstable_customized_partition_wrapperÚcache_on_selfÚcmpÚdevice_need_guardÚget_current_backendÚget_device_tflopsÚget_dtype_sizeÚget_gpu_dram_gbpsÚget_op_namesÚGraphPartitionMapÚIndentedBufferÚis_collectiveÚis_cudagraph_unsafe_opÚis_gpuÚis_multi_outputs_templateÚ#is_output_of_multi_outputs_templateÚis_waitÚsympy_product)ÚVÚfusionÚloop_orderingÚcompute_dependenciesÚ
cudagraphsÚBaseSchedulerNoder   ÚPartitionTypeÚ_TÚ_Pc                  ól   — e Zd ZU dZded<   dZded<   dZded<   d„ Zedd	„«       Z	e	 d	 	 	 dd
„«       Z
y)ÚFusionResultNzbool | NoneÚshould_fusezCallable[[], bool] | NoneÚcallable_fnúLambdaFuture | NoneÚfuturec                óL   — | j                   d u| j                  d uz  sJ d«       ‚y )NzLFusion result should contain either fusion decision or callable_fn, not both)re   rf   ©Úselfs    úX/var/www/html/ia-enhanced/venv/lib/python3.12/site-packages/torch/_inductor/scheduler.pyÚ__post_init__zFusionResult.__post_init__s   s0   € Ø× Ñ ¨Ð,°×1AÑ1AÈÐ1MÒNð 	
ØZó	
ÑNó    c                ó   — t        |¬«      S )N)re   ©rd   )Úclsre   s     rl   ÚfusezFusionResult.fusex   s   € ä¨Ô4Ð4rn   c                ó   — t        ||¬«      S )N©rf   rh   rp   )rq   rf   rh   s      rl   Úfrom_callablezFusionResult.from_callable|   s   € ô ¨¸FÔCÐCrn   )re   Úbool©N)rf   úCallable[[], bool]rh   rg   )Ú__name__Ú
__module__Ú__qualname__re   Ú__annotations__rf   rh   rm   Úclassmethodrr   ru   © rn   rl   rd   rd   m   sf   … à#€K�Ó#Ø-1€KÐ*Ó1Ø"&€FÐÓ&ò
ð
 ò5ó ð5ð àLPðDØ,ðDØ6IòDó ñDrn   rd   c                  óB   — e Zd ZU ded<   ded<   ded<   dZded<   d
d	„Zy)ÚPendingFusionrx   rf   r_   Únode1Únode2Nrg   rh   c                ó2   — | j                   | j                  fS rw   ©r�   r‚   rj   s    rl   Úget_fusion_nodeszPendingFusion.get_fusion_nodesŠ   s   € Ø—
‘
˜DŸJ™JÐ'Ð'rn   )Úreturnú+tuple[BaseSchedulerNode, BaseSchedulerNode])ry   rz   r{   r|   rh   r…   r~   rn   rl   r€   r€   ƒ   s$   … à#Ó#ØÓØÓØ"&€FÐÓ&ô(rn   r€   c                  ó$  — e Zd ZdZedd„«       Zedd„«       Ze	 	 	 	 	 	 dd„«       Zedd„«       Z	e	 	 	 	 	 	 dd„«       Z
e	 	 	 	 	 	 dd„«       Zedd„«       Ze	 	 	 	 	 	 dd	„«       Zedd
„«       Ze	 	 	 	 	 	 dd„«       Zedd„«       Zedd„«       Zy)ÚMixOrderReductionz™
    This class contains utility functions to decide if we should fuse reductions
    reducing across different dimensions of the same input tensor.
    c                óf   — | j                  «       xr  t        d„ | j                  «       D «       «      S )Nc              3  óÀ   K  — | ]V  }t        |t        «      rD|j                  «       r4t        |j                  t        «      r|j                  j
                  d u–— ŒX y ­wrw   )Ú
isinstanceÚSchedulerNodeÚis_reductionÚnoder   Ú_split_size©Ú.0Úsubnodes     rl   ú	<genexpr>z7MixOrderReduction.is_split_reduction.<locals>.<genexpr>–   sK   è ø€ ò +
àÜ˜'¤=Ô1Ø×$Ñ$Ô&Ü˜7Ÿ<™<¬Ô8ð	 �L‰L×$Ñ$¨DÔ0ñ+
ùs   ‚AA)rŽ   ÚallÚ	get_nodes©r�   s    rl   Úis_split_reductionz$MixOrderReduction.is_split_reduction”   s3   € à× Ñ Ó"ò 
¤sñ +
àŸ>™>Ó+ô+
ó (
ð 	
rn   c                ó`  — | j                  |«      �rŽd }d }|j                  «       D �]n  }t        |t        «      r*|j	                  «       rt        |j
                  t        «      sŒ?|j
                  j                  €J ‚t        j                  j                  j                  t        |j
                  j                  «      «      }|j
                  j                  €J ‚t        j                  j                  j                  t        |j
                  j                  «      «      }|€|}|}�Œt        j                  j                  j                  ||«      sJ |› d|› �«       ‚t        j                  j                  j                  ||«      r�ŒeJ |› d|› �«       ‚ |€J ‚||fS |j                  d   S )Nú v.s. r   )r˜   r–   rŒ   r�   rŽ   r�   r   Ú_original_rangesrZ   ÚgraphÚsizevarsÚsimplifyrY   Ú_original_reduction_rangesÚstatically_known_equalsÚgroup)rq   r�   ÚxnumelÚrnumelr“   Ú	curxnumelÚ	currnumels          rl   Úget_numel_rnumelz"MixOrderReduction.get_numel_rnumelž   s‹  € à×!Ñ! $Õ'ØˆFØˆFØŸ>™>Ó+ó 4�ä˜w¬Ô6Ø×,Ñ,Ô.Ü" 7§<¡<´Ô@àà—|‘|×4Ñ4Ð@Ð@Ð@ÜŸG™G×,Ñ,×5Ñ5Ü! '§,¡,×"?Ñ"?Ó@ó�	ð —|‘|×>Ñ>ÐJÐJÐJÜŸG™G×,Ñ,×5Ñ5Ü! '§,¡,×"IÑ"IÓJó�	ð �>Ø&�FØ&’FäŸ7™7×+Ñ+×CÑCØ 	ôð 4à ˜ ¨	 {Ð3ó4ð ô Ÿ7™7×+Ñ+×CÑCØ 	öð 4à ˜ ¨	 {Ð3ó4ð ð14ð8 Ð%Ð%Ð%Ø˜FÐ#Ð#à—:‘:˜a‘=Ð rn   c                óÊ   — | j                  |«      }| j                  |«      }t        |«      dk7  st        |«      dk7  s||k(  ryt        |«      t        t        |«      «      k(  S )Né   F)r¦   ÚlenÚtupleÚreversed)rq   r�   r‚   Úg1Úg2s        rl   Úhas_mix_reduction_ordersz*MixOrderReduction.has_mix_reduction_ordersÄ   sZ   € ð ×!Ñ! %Ó(ˆØ×!Ñ! %Ó(ˆäˆr‹7�aŠ<œ3˜r›7 aš<¨2°ª8Øä�R‹yœE¤(¨2£,Ó/Ñ/Ð/rn   c                óR  — d}|j                   j                  D ]&  }t        |t        «      sŒ|j                  |k(  sŒ$|} n |sy|j
                  }|j                   j                  }|sDt        |t        «      sJ t        |«      › «       ‚|j                  d   j                   j                  }|sJ ‚t        |«      t        |j                  «      z
  syt        j                  j                  j                  t!        |j"                  «      t!        |j%                  «       «      «      ryy)z@
        The access to 'buf' is not a broadcast access.
        NFr   T)Úread_writesÚreadsrŒ   r4   ÚnameÚindexÚ
var_rangesÚFusedSchedulerNodeÚtypeÚsnodesr   r"   rZ   rœ   r�   r    rY   ÚsizeÚvalues)rq   Úbufr�   Ú	found_depÚdepr³   r´   s          rl   Ú_is_full_accessz!MixOrderReduction._is_full_accessÐ   sû   € ð
 ˆ	Ø×#Ñ#×)Ñ)ò 	ˆCÜ˜#œyÕ)¨c¯h©h¸#«oØ�	Ùð	ñ
 Øà—‘ˆØ×%Ñ%×0Ñ0ˆ
áÜ˜dÔ$6Ô7ÐH¼DÀ»J¸<ÓHÐ7ØŸ™ Q™×3Ñ3×>Ñ>ˆJáÐˆzÜ˜:Ó&¬°E×4FÑ4FÓ)GÒGØô
 �7‰7×Ñ×3Ñ3Ü˜)Ÿ.™.Ó)¬=¸×9JÑ9JÓ9LÓ+Mô
ð Ørn   c                óÈ   — g }|j                  «       |j                  «       z  }|D ]9  }| j                  ||«      sŒ| j                  ||«      sŒ)|j                  |«       Œ; |S rw   )Úused_buffer_namesr½   Úappend)rq   r�   r‚   ÚoutÚcommon_readsrº   s         rl   Úget_common_readz!MixOrderReduction.get_common_readò   se   € ð ˆØ×.Ñ.Ó0°5×3JÑ3JÓ3LÑLˆØò 	 ˆCØ×"Ñ" 3¨Õ.°3×3FÑ3FÀsÈEÕ3RØ—
‘
˜3•ð	 ð ˆ
rn   c                ó>   — t        | j                  ||«      «      dkD  S ©Nr   )r©   rÃ   ©rq   r�   r‚   s      rl   Úhas_common_readz!MixOrderReduction.has_common_readþ   s!   € ô �3×&Ñ& u¨eÓ4Ó5¸Ñ9Ð9rn   c                óŒ   — | j                  |«      }t        j                  j                  j	                  |d   |d   z  d¬«      S )Nr   r   ©Úfallback)r¦   rZ   rœ   r�   Úoptimization_hint)rq   r�   r¬   s      rl   Ú	get_numelzMixOrderReduction.get_numel  s>   € à×!Ñ! $Ó'ˆÜ�w‰w×Ñ×1Ñ1°"°Q±%¸"¸Q¹%±-È!Ð1ÓLÐLrn   c                ó$   — | j                  |«      S rw   )rÌ   rÆ   s      rl   Úget_fusion_scorez"MixOrderReduction.get_fusion_score	  s   € ð
 �}‰}˜UÓ#Ð#rn   c                óx  — t         j                  j                  syt        j                  j
                  ry|j                  «       r|j                  «       sy|j                  «       j                  }|dvst        |«      dk7  ry|j                  «       r|j                  «       sy|j                  |j                  «       z  s|j                  |j                  «       z  ry| j                  ||«      syt        j                  ||«      }t!        |«      dk(  ry| j#                  |«      r||}}n| j#                  |«      r||}}ny| j%                  |«      }|\  }}	t         j                  j&                  sÂd}
t        j                  j(                  j+                  t-        j.                  ||	z  |
«      «      syt        j                  j(                  j+                  t-        j.                  ||	dz  «      «      syt        j                  j(                  j+                  t-        j.                  |d«      «      syt1        d„ |j3                  «       D «       «      ryt        j                  j(                  j5                  |	d	«      syt        j7                  |«      ryt9        d
„ |j3                  «       D «       «      }|S )zP
        Check whether we can fuse two reductions with mix loop orders.
        F)ÚcudaÚxpuÚtritonr   i  P r¨   i   c              3  ó¼   K  — | ]T  }|j                  «       rB|j                  j                  j                  t        j
                  t        j                  fv–— ŒV y ­wrw   )rŽ   r�   ÚdataÚreduction_hintrD   ÚINNERÚDEFAULTr‘   s     rl   r”   z-MixOrderReduction.can_fuse.<locals>.<genexpr>[  sR   è ø€ ò 
ð Ø×#Ñ#Ô%ð �L‰L×Ñ×,Ñ,ä×#Ñ#Ü×%Ñ%ðôñ
ùs   ‚AAi @  c              3  ót   K  — | ]0  }|j                  «       r|j                  j                  «       d v –— Œ2 y­w)>   ÚsumÚprodN)rŽ   r�   Úget_reduction_typer‘   s     rl   r”   z-MixOrderReduction.can_fuse.<locals>.<genexpr>q  s=   è ø€ ò 
ð Ø×#Ñ#Ô%ð �L‰L×+Ñ+Ó-ðôñ
ùs   ‚68)r(   rÒ   Úmix_order_reductionrZ   rœ   Úcpp_wrapperrU   Ú
get_devicer¶   rL   rŽ   Ú	ancestorsÚget_operation_namesr®   r‰   rÃ   r©   Úis_contiguous_noder¦   Ú#mix_order_reduction_non_strict_moder�   Úguard_or_trueÚsympyÚGeÚanyr–   Ústatically_known_leqr˜   r•   )rq   r�   r‚   Údevice_typerÂ   Úcontiguous_nodeÚ
other_noder¬   ÚnrowÚncolÚ
size_thresrÁ   s               rl   Úcan_fusezMixOrderReduction.can_fuse  sJ  € ô
 �}‰}×0Ò0Øô �7‰7×ÒØà�|‰|Œ~ U§\¡\¤^ØØ×&Ñ&Ó(×-Ñ-ˆà˜Ñ.Ü" ;Ó/°8Ò;àØ×!Ñ!Ô#¨5×+=Ñ+=Ô+?Øà�O‰O˜e×7Ñ7Ó9Ò9Ø�O‰O˜e×7Ñ7Ó9Ò9ð ð ×+Ñ+¨E°5Ô9Øô )×8Ñ8¸ÀÓFˆÜˆ|Ó Ò!Øà×!Ñ! %Ô(Ø*/°˜Z‰OØ×#Ñ# EÔ*Ø*/°˜Z‰Oàà×!Ñ! /Ó2ˆØ‰
ˆˆdô �}‰}×@Ò@ð #ˆJô
 —7‘7×#Ñ#×1Ñ1´%·(±(¸4À$¹;È
Ó2SÔTØô
 —7‘7×#Ñ#×1Ñ1´%·(±(¸4ÀÈÁÓ2JÔKØô
 —7‘7×#Ñ#×1Ñ1´%·(±(¸4ÀÓ2FÔGØô ñ 
ð +×4Ñ4Ó6ô
ô 
ð ô
 �w‰w×Ñ×4Ñ4°T¸9ÔEØä×/Ñ/°Ô@Øô ñ 
ð &×/Ñ/Ó1ô
ó 
ˆð ˆ
rn   c                ó&   — | j                  ||«      S rw   )rî   rÆ   s      rl   Úare_mix_order_reductionsz*MixOrderReduction.are_mix_order_reductions|  s   € ð �|‰|˜E 5Ó)Ð)rn   c                ó\   ‡ ‡— t        ˆ ˆfd„‰j                  j                  D «       «      syy)Nc              3  óV   •K  — | ]   }‰j                  |j                  ‰«      –— Œ" y ­wrw   )Úis_contiguous_loadr²   )r’   r¼   rq   r�   s     €€rl   r”   z7MixOrderReduction.is_contiguous_node.<locals>.<genexpr>„  s'   øè ø€ ò 
Ø7:ˆC×"Ñ" 3§8¡8¨T×2ñ
ùó   ƒ&)FT)r•   r°   r±   )rq   r�   s   ``rl   rá   z$MixOrderReduction.is_contiguous_node‚  s,   ù€ äô 
Ø>B×>NÑ>N×>TÑ>Tô
ô 
ð Ørn   c                ó  — ddl m} |j                  «       D ]ë  }t        |t        «      sJ ‚|j
                  }|j                  |j                     }|D �cg c]  }|j                  |k(  sŒ|j                  ‘Œ  }}t        |«      dk(  rŒr|D ]u  }	|j                  |	   }
|j                  }t        |j                  «       «      }t        j                   j"                  j%                  |
||«      }|d   dk(  rŒk|d   dk(  rŒt  y Œí yc c}w )Nr   )ÚMemoryUsageTypeéÿÿÿÿr   FT)Útorch._inductor.loop_bodyrö   r–   rŒ   r�   Ú_bodyÚmemory_usageÚLOADÚbuffer_nameÚ
index_namer©   Úindexing_exprsr´   ÚlistÚkeysrZ   rœ   r�   Ústride_vars)rq   rº   Úparent_noderö   r�   Ú	loop_bodyÚentriesÚeÚindex_namesrý   Ú
index_exprr´   Úvar_symbolsr  s                 rl   ró   z$MixOrderReduction.is_contiguous_loadŠ  s  € å=à×)Ñ)Ó+ò 	!ˆDÜ˜d¤MÔ2Ð2Ð2ØŸ
™
ˆIØ×,Ñ,¨_×-AÑ-AÑBˆGØ18ÖQ¨A¸A¿M¹MÈSÓ<P˜1Ÿ<›<ÐQˆKÐQä�;Ó 1Ò$Øð *ò !�
Ø&×5Ñ5°jÑA�
Ø&×1Ñ1�
ô # :§?¡?Ó#4Ó5�ÜŸg™g×.Ñ.×:Ñ:ØØØó�ð $ B™¨1Ó,°¸B±À1Ó0DÚ ñ!ð	!ð2 ùò+ Rs   ÁDÁ*DN©r�   r_   r†   rv   )r�   r_   r†   ztuple[sympy.Expr, sympy.Expr]©r�   r_   r‚   r_   r†   rv   )rº   Ústrr�   r_   r†   rv   )r�   r_   r‚   r_   r†   ú	list[str])r�   r_   r†   Úint©r�   r_   r‚   r_   r†   r  )rº   r  r  r_   r†   rv   )ry   rz   r{   Ú__doc__Ústaticmethodr˜   r}   r¦   r®   r½   rÃ   rÇ   rÌ   rÎ   rî   rð   rá   ró   r~   rn   rl   r‰   r‰   Ž   sv  „ ñð
 ò
ó ð
ð ò#!ó ð#!ðJ ð	0Ø%ð	0Ø.?ð	0à	ò	0ó ð	0ð òó ððB ð	Ø%ð	Ø.?ð	à	ò	ó ð	ð ð:Ø%ð:Ø.?ð:à	ò:ó ð:ð
 òMó ðMð ð$Ø%ð$Ø.?ð$à	ò$ó ð$ð òhó ðhðT ð*Ø%ð*Ø.?ð*à	ò*ó ð*ð
 òó ðð òó ñrn   r‰   c                  óÜ   — e Zd ZU ded<   ded<   ded<    ej
                  e¬«      Zded	<    ej
                  e¬«      Z	d
ed<   dd„Z
dd„Zdd„Zdd„Zdd„Zdd„Zdd„Zdd„Zdd„Zdd„Zy)ÚSchedulerBufferÚ	SchedulerÚ	schedulerz	ir.Bufferr�   úBaseSchedulerNode | NoneÚdefining_op)Údefault_factoryúlist[NodeUser]ÚusersrA   Ú
mpi_bufferc                óB   — | j                   }|€J ‚|j                  «       S rw   )r  Úget_name)rk   Úops     rl   Údefining_op_namez SchedulerBuffer.defining_op_name´  s#   € Ø×ÑˆØˆ~Ðˆ~Ø�{‰{‹}Ðrn   c                ó@   — t        | j                  j                  «      S rw   )Úhashr�   r²   rj   s    rl   Ú__hash__zSchedulerBuffer.__hash__¹  s   € Ü�D—I‘I—N‘NÓ#Ð#rn   c                óv  — t        «       }| j                  «       }|j                  |› dt        | j                  «      j
                  › �«       |j                  |› d| j                  j                  › �«       | j                  «       r-|j                  |› dt        | j                  «       «      › �«       | j                  «       r-|j                  |› dt        | j                  «       «      › �«       t        | j                  «      dk  r0|j                  |› d| j                  › �«       |j                  «       S |j                  |› d�«       |j                  d«      5  | j                  D ]  }|j                  |› d�«       Œ 	 d d d «       |j                  d	«       |j                  «       S # 1 sw Y   Œ*xY w)
Nú: z
.layout = z.aliases = z.mutations = r   z	.users = z
.users = [ú,ú])rR   r  Ú	writeliner¶   r�   ry   ÚlayoutÚget_aliasesÚpformatÚget_mutationsr©   r  ÚindentÚgetrawvalue)rk   Úresultr²   Úusers       rl   Ú	debug_strzSchedulerBuffer.debug_str¼  s…  € ÜÓ!ˆØ�}‰}‹ˆØ×Ñ˜D˜6 ¤D¨¯©£O×$<Ñ$<Ð#=Ð>Ô?Ø×Ñ˜D˜6 ¨D¯I©I×,<Ñ,<Ð+=Ð>Ô?Ø×ÑÔØ×Ñ ˜v [´¸×9IÑ9IÓ9KÓ1LÐ0MÐNÔOØ×ÑÔØ×Ñ ˜v ]´7¸4×;MÑ;MÓ;OÓ3PÐ2QÐRÔSäˆt�z‰z‹?˜aÒØ×Ñ ˜v Y¨t¯z©z¨lÐ;Ô<ð ×!Ñ!Ó#Ð#ð ×Ñ ˜v ZÐ0Ô1Ø—‘˜qÓ!ñ 1Ø ŸJ™Jò 1�DØ×$Ñ$¨ v¨Q ZÕ0ñ1÷1ð ×Ñ˜SÔ!Ø×!Ñ!Ó#Ð#÷	1ð 1ús   Å&F/Æ/F8c                ó6   — | j                   j                  «       S rw   ©r�   r  rj   s    rl   r  zSchedulerBuffer.get_nameÐ  ó   € Ø�y‰y×!Ñ!Ó#Ð#rn   c                óö  — | j                   €J ‚| j                   j                  «       sy | j                   j                  «       sL| j                   j                  «       s2t	        | j                   j                  «       t        j                  «      r4t        j                  j                  j                  | j                   «       y t        t        j                  d«      ré| j                  «       t        j                  j                  v r¿t        j                  j                  | j                  «          }|| j                   j"                  v r$| j                   j"                  |   j                   }n#| j                   j$                  |   j                   }t        j                  j                  j'                  || j                   «       y t        j                  j                  j                  | j                   «       y )NÚargs)r�   Úshould_allocateÚget_inputs_that_alias_outputÚget_mutation_namesrŒ   Úget_output_specr+   ÚCommBufferLayoutrZ   rœ   Úwrapper_codeÚcodegen_allocationÚhasattrÚkernelr  Úinplace_update_buffersr  Úname_to_donated_bufferÚname_to_bufÚcodegen_inplace_reuse)rk   Úinput_buffer_nameÚinput_buffers      rl   ÚallocatezSchedulerBuffer.allocateÓ  sV  € Ø�y‰yÐ$Ð$Ð$Ø�y‰y×(Ñ(Ô*Øð �I‰I×2Ñ2Ô4Ø�y‰y×+Ñ+Ô-Ü˜$Ÿ)™)×3Ñ3Ó5´r×7JÑ7JÔKä�G‰G× Ñ ×3Ñ3°D·I±IÔ>Øô ”A—H‘H˜fÔ%Ø—‘“¤1§8¡8×#BÑ#BÑBô !"§¡× ?Ñ ?ÀÇÁÃÑ PÐØ  D§N¡N×$IÑ$IÑIØ#Ÿ~™~×DÑDØ%ñ ç‘$ñ ð  $Ÿ~™~×9Ñ9Ð:KÑL×QÑQ�Ü�G‰G× Ñ ×6Ñ6ØØ—	‘	õô
 �G‰G× Ñ ×3Ñ3°D·I±IÕ>rn   c                ó  — | j                   €J ‚t        | j                   j                  t        j                  «      st        | j                   «      ry| j                  D ]  }t        |j                   t        «      sŒ y y©NFT)r�   rŒ   r'  r+   r?   rV   r  Ú
OutputNode)rk   Úuses     rl   Úcan_freezSchedulerBuffer.can_freeô  sg   € à�y‰yÐ$Ð$Ð$Ü�d—i‘i×&Ñ&¬¯©Ô6Ô:SØ�I‰Iô;
ð Ø—:‘:ò 	ˆCÜ˜#Ÿ(™(¤JÕ/Ùð	ð rn   c                ó,  — i }|D ]o  }t        |j                  «      |v r>|j                  |t        |j                  «         «      |t        |j                  «      <   ŒX||t        |j                  «      <   Œq t        |j	                  «       «      | _        y rw   )Úidr�   Úmergerÿ   r¹   r  )rk   r  r-  rH  s       rl   Ú	set_userszSchedulerBuffer.set_users   st   € à&(ˆØò 	+ˆCÜ�#—(‘(‹|˜vÑ%Ø'*§y¡y°¼¸3¿8¹8»Ñ1EÓ'F�”r˜#Ÿ(™(“|Ò$à'*�”r˜#Ÿ(™(“|Ò$ð		+ô
 ˜&Ÿ-™-›/Ó*ˆ�
rn   c                óR   — | j                   €J ‚| j                   j                  «       S rw   )r�   r6  rj   s    rl   r(  zSchedulerBuffer.get_aliases
  s%   € Ø�y‰yÐ$Ð$Ð$Ø�y‰y×5Ñ5Ó7Ð7rn   c                óR   — | j                   €J ‚| j                   j                  «       S rw   )r�   r7  rj   s    rl   r*  zSchedulerBuffer.get_mutations  ó%   € Ø�y‰yÐ$Ð$Ð$Ø�y‰y×+Ñ+Ó-Ð-rn   c                óR   — | j                   j                  «       j                  «       S rw   )r�   r8  rÞ   rj   s    rl   rÞ   zSchedulerBuffer.get_device  s   € Ø�y‰y×(Ñ(Ó*×5Ñ5Ó7Ð7rn   N©r†   r  ©r†   r  ©r†   ÚNone©r†   rv   )r  r  r†   rU  ©r†   zSequence[str]©r†   útorch.device | None)ry   rz   r{   r|   ÚdataclassesÚfieldrÿ   r  rA   r  r  r!  r/  r  rD  rI  rM  r(  r*  rÞ   r~   rn   rl   r  r  ª  sz   … àÓØ
ƒOØ)Ó)Ø-˜K×-Ñ-¸dÔC€Eˆ>ÓCØ.?¨k×.?Ñ.?Ø3ô/€JÐ+ó óó
$ó$ó($ó?óB
ó+ó8ó.ô8rn   r  c                  ó   — e Zd ZU dZded<   y)ÚSchedulerDonatedBufferNr  r  )ry   rz   r{   r  r|   r~   rn   rl   r]  r]    s   … à,0€KÐ)Ô0rn   r]  c                  ó  — e Zd ZU ded<   ded<   ded<   ded<   ded<   ded	<   ded
<   ded<   ded<   dZded<   ded<   ded<   dZded<   ded<   ded<   dZded<   dTd„ZdUd „ZdVd!„Z	dVd"„Z
dVd#„ZdWd$„ZdVd%„ZdXd&„Z	 	 	 	 	 	 dYd'„ZdZd(„Zd[d)„Zd\d*„Zd]d+„Z	 	 	 	 	 	 d^d,„ZdXd-„Zd_d.„Zd_d/„ZdXd0„ZdXd1„Z	 	 	 	 d`d2„ZdVd3„ZdVd4„Zed_d5„«       Zed_d6„«       Zed\d7„«       Z ed\d8„«       Z!dad9„Z"dbd:„Z#dcd;„Z$ddd<„Z%d\d=„Z&d\d>„Z'd\d?„Z(d\d@„Z)d\dA„Z*d\dB„Z+d\dC„Z,d\dD„Z-dedE„Z.d\dF„Z/dXdG„Z0	 df	 	 	 	 	 dgdH„Z1edhdI„«       Z2edhdJ„«       Z3edhdK„«       Z4	 	 	 	 	 	 didL„Z5	 	 	 	 	 	 djdM„Z6edkdN„«       Z7dldO„Z8edldP„«       Z9dmdQ„Z:dndR„Z;e<	 	 	 	 dodS„«       Z=y)pr_   úOrderedSet[str]rß   z7tuple[torch.device, tuple[tuple[sympy.Expr, ...], ...]]r¡   Ú
last_usager  Úmin_input_distanceÚmax_input_distanceÚ	min_orderÚ	max_orderrB   Úmpi_nodeúdict[str, str]Úmutation_renamesNúir.Operation | Noner�   úlist[SchedulerBuffer]Úoutputsúdict[str, SchedulerBuffer]Úoutputs_by_nameúfloat | NoneÚoverride_estimated_runtimeúdependencies.ReadWritesr°   úOrderedSet[Dep]Úunmet_dependenciesFrv   Úwrittenc                ó"   — || _         d„ | _        y )Nc                 ó   — g S rw   r~   )r4  Úkwargss     rl   ú<lambda>z,BaseSchedulerNode.__init__.<locals>.<lambda>5  s   €  B€ rn   )r  Údebug_device_str)rk   r  s     rl   Ú__init__zBaseSchedulerNode.__init__2  s   € Ø$-ˆŒá&ð 	Õrn   c                óv  — || _         t        «       | _        d| _        d| _        t        t
           «       | _        d| _        |j                  «       D �cg c]  }t        | j                  || ¬«      ‘Œ c}| _        | j                  D �ci c]  }|j                  «       |“Œ c}| _        i | _        y c c}w c c}w )Nr   F)r  r�   r  )r�   r   rß   ra  rb  r  r`  rr  Úget_outputsr  r  rj  r  rl  rg  )rk   r�   Úoutputrº   s       rl   Ú_init_from_nodez!BaseSchedulerNode._init_from_node8  s³   € ØˆŒ	Ü#›ˆŒØ"#ˆÔØ"#ˆÔÜ$Üñ
ó ˆŒð ˆŒð ×*Ñ*Ó,ö
ð ô ØŸ.™.ØØ öò
ˆŒð @D¿|¹|ÖL¸ §¡£°Ñ 3ÒLˆÔð !#ˆÕùò
ùò  Ms   ÁB1Â	B6c                óT   — t        | «      j                  › d| j                  «       ›d�S )Nz(name=ú)©r¶   ry   r  rj   s    rl   Ú__repr__zBaseSchedulerNode.__repr__R  s'   € Ü�t“*×%Ñ%Ð& f¨T¯]©]«_Ð,?¸qÐAÐArn   c                óˆ  — | j                  «       }t        «       }|j                  |› dt        | «      j                  › dt        t        | dd«      «      j                  › d|› dt        | j                  j                  «      › d|› dt        | j                  «      › d|› d	t        | j                  j                  | j                  z
  «      › d|› d
| j                  › d|› d| j                  › d|› d�«       |j                  «       5  | j                  «       D ]!  }|j                  |j                  «       «       Œ# 	 ddd«       |j!                  d«       	 |j                  | j#                  «       «       |j+                  «       j-                  «       S # 1 sw Y   ŒXxY w# t$        $ r t&        j)                  dd¬«       Y ŒLw xY w)ú#Longer form printout for trace logsr#  ú(r�   Nú)
ú
.writes = ú
ú.unmet_dependencies = ú.met_dependencies = ú.min_input_distance = ú.max_input_distance = z.outputs = [
        r%  úIgnoring error in debug_str()T©Úexc_info)r  rR   Úsplicer¶   ry   Úgetattrr)  r°   Úwritesrq  r±   ra  rb  r+  rz  r/  r&  Údebug_str_extraÚ	ExceptionÚlogÚwarningr,  Úrstrip)rk   r²   rº   rÁ   s       rl   r/  zBaseSchedulerNode.debug_strU  s³  € à�}‰}‹ˆÜÓˆØ�
‰
à€€bŒˆd‹×	Ñ	Ð˜Qœt¤G¨D°&¸$Ó$?Ó@×IÑIÐJð KØ€€j”˜×)Ñ)×0Ñ0Ó1Ð2ð 3Ø€ÐœW T×%<Ñ%<Ó=Ð>ð ?Ø€Ðœ7 4×#3Ñ#3×#9Ñ#9¸D×<SÑ<SÑ#SÓTÐUð VØ€Ð˜T×4Ñ4Ð5ð 6Ø€Ð˜T×4Ñ4Ð5ð 6Ø€ð 	ðô
	
ð �Z‰Z‹\ñ 	,Ø×'Ñ'Ó)ò ,�Ø—
‘
˜3Ÿ=™=›?Õ+ñ,÷	,ð 	�‰�cÔð	HØ�J‰J�t×+Ñ+Ó-Ô.ð �‰Ó ×'Ñ'Ó)Ð)÷	,ð 	,ûô ò 	HÜ�K‰KÐ7À$ˆKÖGð	Hús   Ä5FÅF ÆFÆ GÇ Gc                 ó   — y)NÚ r~   rj   s    rl   r‘  z!BaseSchedulerNode.debug_str_extrap  ó   € Ørn   c                ó$   — | j                  | «      S rw   )rw  rj   s    rl   Ú_debug_str_for_devicez'BaseSchedulerNode._debug_str_for_devices  s   € Ø×$Ñ$ TÓ*Ð*rn   c                ó°  — t        | j                  dd «      }d}t        |t        j                  j
                  j                  «      r'd|j                  |j                  «       gdd¬«      z   }nct        |t        j                  j
                  j                  «      r5d|j                  |j                  «       |j                  «       gdd¬«      z   }| › |› �S )NrÔ   r—  ú, F)ÚshortenÚ	multiline)r�  r�   rŒ   ÚtorchÚ	_inductorr+   r   Ú
str_helperÚget_sizeÚ	ReductionÚget_reduction_sizerÛ   )rk   Ú
maybe_dataÚdata_strs      rl   Údebug_str_shortz!BaseSchedulerNode.debug_str_shortv  sË   € Ü˜TŸY™Y¨°Ó5ˆ
ØˆÜ�j¤%§/¡/×"4Ñ"4×">Ñ">Ô?Ø˜j×3Ñ3Ø×$Ñ$Ó&Ð'°À%ð 4ó ñ ‰Hô ˜
¤E§O¡O×$6Ñ$6×$@Ñ$@ÔAØ˜j×3Ñ3Ø×.Ñ.Ó0°*×2OÑ2OÓ2QÐRØØð 4ó ñ ˆHð
 �˜�zÐ"Ð"rn   c                óp   — t         j                  d| | j                  | j                  j                  «       y )Nz(%s: unmet_dependencies = %s, writes = %s)r“  Úinforq  r°   r�  rj   s    rl   Úlog_detailszBaseSchedulerNode.log_details…  s,   € Ü�‰Ø6ØØ×#Ñ#Ø×Ñ×#Ñ#õ		
rn   c                 ó   — y©NFr~   )rk   Úself_depÚ	other_deps      rl   Úreorder_loops_by_dep_pairz+BaseSchedulerNode.reorder_loops_by_dep_pair�  ó   € ð rn   c                óî   — d„ | j                   j                  «       D «       D �ci c]  }||v r|||   “Œ c}| _        | j                  | j                   j	                  | j                  «      «       y c c}w )Nc              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   ©r²   ©r’   r¼   s     rl   r”   z9BaseSchedulerNode.update_mutated_names.<locals>.<genexpr>•  s   è ø€ ÒQ c˜Ÿ�ÑQùó   ‚)r°   Úreads_and_writesrg  Úset_read_writesÚrename)rk   Úrenamesr²   s      rl   Úupdate_mutated_namesz&BaseSchedulerNode.update_mutated_names’  sp   € ñ R¨T×-=Ñ-=×-NÑ-NÓ-PÔQö!
àØ�w‰ð �'˜$‘-Ñò!
ˆÔð
 	×Ñ˜T×-Ñ-×4Ñ4°T×5JÑ5JÓKÕLùò!
s   ¤A2c                óX   — | j                  | j                  j                  |«      «       y rw   )r·  r°   Ú	with_read©rk   r¼   s     rl   Úadd_fake_depzBaseSchedulerNode.add_fake_depš  s!   € Ø×Ñ˜T×-Ñ-×7Ñ7¸Ó<Õ=rn   c                óB   — t        d„ | j                  «       D «       «      S )Nc              3  ó`   K  — | ]&  }|j                  «       xs |j                  «       –— Œ( y ­wrw   )r(  r*  ©r’   rº   s     rl   r”   z=BaseSchedulerNode.has_aliasing_or_mutation.<locals>.<genexpr>ž  s-   è ø€ ò 
Ø9<ˆC�O‰OÓÒ4 ×!2Ñ!2Ó!4Ó4ñ
ùs   ‚,.)ræ   rz  rj   s    rl   Úhas_aliasing_or_mutationz*BaseSchedulerNode.has_aliasing_or_mutation�  s%   € Üñ 
Ø@D×@PÑ@PÓ@Rô
ó 
ð 	
rn   c                óh   — || _         | j                   j                  | _        | j                  «        y rw   )r°   r±   rq  Ú
prune_deps)rk   Úrws     rl   r·  z!BaseSchedulerNode.set_read_writes¢  s(   € ØˆÔØ"&×"2Ñ"2×"8Ñ"8ˆÔØ�‰Õrn   c                ób   ‡— | j                  «       }t        ˆfd„|D «       «      }||z
  | _        y )Nc              3  óB   •K  — | ]  }‰j                  ||«      –— Œ y ­wrw   )Úget)r’   ÚkÚmutation_real_names     €rl   r”   z3BaseSchedulerNode.set_last_usage.<locals>.<genexpr>«  s   øè ø€ Ò!UÀ1Ð"4×"8Ñ"8¸¸A×">Ñ!Uùó   ƒ)Úused_or_aliased_buffer_namesr   r`  )rk   Úfuture_used_buffersrÊ  Úused_bufferss     ` rl   Úset_last_usagez BaseSchedulerNode.set_last_usage§  s0   ø€ ð ×8Ñ8Ó:ˆÜ!Ó!UÈÔ!UÓUˆØ&Ð)<Ñ<ˆ�rn   c                óF   — | j                   D ]  }|j                  «        Œ y rw   )rj  rD  )rk   rº   s     rl   Úmark_runzBaseSchedulerNode.mark_run®  s   € Ø—<‘<ò 	ˆCØ�L‰L�Nñ	rn   c                óž   — t        d„ t        j                  | j                  j                  | j                  j
                  «      D «       «      S )Nc              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   z6BaseSchedulerNode.used_buffer_names.<locals>.<genexpr>³  s   è ø€ ò 
àð �H�Hñ
ùrµ  )r   Ú	itertoolsÚchainr°   r±   r�  rj   s    rl   r¿   z#BaseSchedulerNode.used_buffer_names²  s?   € Üñ 
ä —‘ t×'7Ñ'7×'=Ñ'=¸t×?OÑ?O×?VÑ?VÓWô
ó 
ð 	
rn   c                ó\  ‡— t        «       Št        j                  | j                  j                  | j                  j
                  «      D �cg c]*  }t        |t        «      r|j                  s|j                  ‘Œ, }}t        |«      dkD  rœ|j                  «       }‰j                  |«       t        j                  j                  j!                  |«      rC|j#                  ˆfd„t        j                  j                  |   j%                  «       D «       «       t        |«      dkD  rŒœ‰S c c}w )zË
        Returns buffer names used by this node, including aliases.

        Note: is_fake WeakDeps are excluded since they are purely for ordering
        and should not affect buffer lifetime.
        r   c              3  ó*   •K  — | ]
  }|‰vr|–— Œ y ­wrw   r~   )r’   ÚaliasÚ
used_namess     €rl   r”   zABaseSchedulerNode.used_or_aliased_buffer_names.<locals>.<genexpr>Ê  s#   øè ø€ ò àð  JÑ.ô	 ñùs   ƒ)r   rÔ  rÕ  r°   r±   r�  rŒ   r6   Úis_faker²   r©   ÚpopÚaddrZ   rœ   Úname_to_bufferrÈ  Úextendr6  )rk   r¼   ÚdepsrÙ  s      @rl   rÌ  z.BaseSchedulerNode.used_or_aliased_buffer_names¸  sì   ø€ ô '1£lˆ
ô !—‘ t×'7Ñ'7×'=Ñ'=¸t×?OÑ?O×?VÑ?VÓWö
àÜ˜s¤GÔ,°·²ð �H‹Hð
ˆð 
ô
 �$‹i˜!ŠmØ—(‘(“*ˆCØ�N‰N˜3ÔÜ�w‰w×%Ñ%×)Ñ)¨#Ô.Ø—‘ó ä!"§¡×!7Ñ!7Øñ"ç2Ñ2Ó4ô	ô ô	 �$‹i˜!‹mð Ðùò!
s   Á/D)c                óL   ‡ — t        ˆ fd„‰ j                  D «       «      ‰ _        y )Nc              3  óf   •K  — | ](  }|j                   ‰j                  j                  vr|–— Œ* y ­wrw   )r²   r  Úavailable_buffer_names©r’   r¼   rk   s     €rl   r”   z/BaseSchedulerNode.prune_deps.<locals>.<genexpr>Ô  s/   øè ø€ ò -
àØ�x‰x˜tŸ~™~×DÑDÑDô ñ-
ùs   ƒ.1©r   rq  rj   s   `rl   rÄ  zBaseSchedulerNode.prune_depsÓ  s#   ø€ Ü",ó -
à×.Ñ.ô-
ó #
ˆÕrn   c                ó¸   ‡ ‡— dˆ fd„Št        ˆfd„‰ j                  j                  D «       «      }‰ j                  ‰ j                  j	                  |«      «       y )Nc                ó  •— t        | t        «      sy| j                  ‰j                  j                  vry‰j                  j                  | j                     j                  «       }|t        j                  j                  v S r¬  )	rŒ   r6   r²   r  r@  r  rZ   rœ   Úremoved_operations)r¼   Úop_namerk   s     €rl   Úshould_prunez7BaseSchedulerNode.prune_weak_deps.<locals>.should_pruneÜ  s_   ø€ Ü˜c¤7Ô+ØØ�x‰x˜tŸ~™~×9Ñ9Ñ9Øà—n‘n×0Ñ0°·±Ñ:×KÑKÓMˆGØœaŸg™g×8Ñ8Ð8Ð8rn   c              3  ó4   •K  — | ]  } ‰|«      sŒ|–— Œ y ­wrw   r~   ©r’   r¼   ré  s     €rl   r”   z4BaseSchedulerNode.prune_weak_deps.<locals>.<genexpr>å  s   øè ø€ ò 
Ø±\À#Õ5FŒCñ
ùó   ƒ‘©r¼   r3   r†   rv   )r   r°   r±   r·  Úremove_reads)rk   Ú	to_removeré  s   ` @rl   Úprune_weak_depsz!BaseSchedulerNode.prune_weak_depsÚ  sN   ù€ õ	9ô ó 
Ø×+Ñ+×1Ñ1ô
ó 
ˆ	ð 	×Ñ˜T×-Ñ-×:Ñ:¸9ÓEÕFrn   c                óF   — t        | || j                  j                  «       y rw   )Ú_prune_redundant_depsr  r@  )rk   Úname_to_fused_nodes     rl   Úprune_redundant_depsz&BaseSchedulerNode.prune_redundant_depsê  s   € ô 	˜dÐ$6¸¿¹×8RÑ8RÕSrn   c                óR   — | j                   €J ‚| j                   j                  «       S rw   )r�   Úget_operation_namerj   s    rl   r  zBaseSchedulerNode.get_nameï  rP  rn   c                ó"   — | j                  «       S rw   ©r  rj   s    rl   Úget_first_namez BaseSchedulerNode.get_first_nameó  s   € Ø�}‰}‹Ðrn   c                óB   — t        d„ | j                  «       D «       «      S )Nc              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   rø  ©r’   r�   s     rl   r”   z8BaseSchedulerNode.get_operation_names.<locals>.<genexpr>ø  s   è ø€ ÒG¨d˜$Ÿ-™-Ÿ/ÑGùó   ‚)r   r–   rj   s    rl   rà   z%BaseSchedulerNode.get_operation_namesö  s   € äÑG°d·n±nÓ6FÔGÓGÐGrn   c                ó:   — t        d„ | j                  D «       «      S )Nc              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   rø  ©r’   rÁ   s     rl   r”   z5BaseSchedulerNode.get_buffer_names.<locals>.<genexpr>ü  s   è ø€ ÒA¨S˜#Ÿ,™,Ÿ.ÑAùrý  )r   rj  rj   s    rl   Úget_buffer_namesz"BaseSchedulerNode.get_buffer_namesú  s   € äÑA°D·L±LÔAÓAÐArn   c                óB   — t        d„ | j                  «       D «       «      S )Nc              3  óZ   K  — | ]#  }t        |t        «      xr t        |d ¬«      –— Œ% y­w)T)Údisallow_fp32_opsN©rŒ   r�   r-   ©r’   Úns     rl   r”   zABaseSchedulerNode.can_codegen_in_low_precision.<locals>.<genexpr>   s7   è ø€ ò 
ð ô �qœ-Ó(ò GÜ+¨AÀÔFóGñ
ùs   ‚)+©r•   r–   rj   s    rl   Úcan_codegen_in_low_precisionz.BaseSchedulerNode.can_codegen_in_low_precisionþ  s%   € äñ 
ð —^‘^Ó%ô
ó 
ð 	
rn   c                óB   — t        d„ | j                  «       D «       «      S )Nc              3  óV   K  — | ]!  }t        |t        «      xr t        |«      –— Œ# y ­wrw   r  r  s     rl   r”   z@BaseSchedulerNode.can_codegen_without_upcasts.<locals>.<genexpr>  s-   è ø€ ò 
àô �qœ-Ó(ÒKÔ-HÈÓ-KÓKñ
ùs   ‚')r  rj   s    rl   r-   z-BaseSchedulerNode.can_codegen_without_upcasts  s#   € äñ 
à—^‘^Ó%ô
ó 
ð 	
rn   c                ó   — | gS rw   r~   rj   s    rl   r–   zBaseSchedulerNode.get_nodes  s	   € Øˆvˆrn   c                ó   — | j                   S rw   )rj  rj   s    rl   rz  zBaseSchedulerNode.get_outputs  s   € Ø�|‰|Ðrn   c                ó    — | j                   |   S rw   )rl  )rk   Úbuf_names     rl   Ú
get_outputzBaseSchedulerNode.get_output  s   € Ø×#Ñ# HÑ-Ð-rn   c                óR   — | j                   €J ‚| j                   j                  «       S rw   )r�   rÞ   rj   s    rl   rÞ   zBaseSchedulerNode.get_device  s%   € Ø�y‰yÐ$Ð$Ð$Ø�y‰y×#Ñ#Ó%Ð%rn   c                óL   — | j                  «       }|d uxr |j                  dk(  S ©NÚcpu)rÞ   r¶   ©rk   Údevices     rl   Úis_cpuzBaseSchedulerNode.is_cpu  s'   € Ø—‘Ó"ˆØ˜TÐ!Ò: f§k¡k°UÑ&:Ð:rn   c                óX   — | j                  «       }|d uxr t        |j                  «      S rw   )rÞ   rU   r¶   r  s     rl   rU   zBaseSchedulerNode.is_gpu  s'   € Ø—‘Ó"ˆØ˜TÐ!Ò9¤f¨V¯[©[Ó&9Ð9rn   c                 ó   — yr¬  r~   rj   s    rl   rŽ   zBaseSchedulerNode.is_reduction"  ó   € Ørn   c                 ó   — yr¬  r~   rj   s    rl   Úis_native_matmulz"BaseSchedulerNode.is_native_matmul%  r  rn   c                 ó   — yr¬  r~   rj   s    rl   Úis_split_scanzBaseSchedulerNode.is_split_scan(  r  rn   c                 ó   — yr¬  r~   rj   s    rl   Úis_templatezBaseSchedulerNode.is_template+  r  rn   c                 ó   — yr¬  r~   rj   s    rl   Ú	is_externzBaseSchedulerNode.is_extern.  r  rn   c                 ó   — yr¬  r~   rj   s    rl   Ú
is_foreachzBaseSchedulerNode.is_foreach1  r  rn   c                 ó   — yr¬  r~   ©rk   Úread_deps     rl   Úcan_inplacezBaseSchedulerNode.can_inplace4  r  rn   c                 ó   — yr¬  r~   rj   s    rl   Úhas_side_effectsz"BaseSchedulerNode.has_side_effects7  r  rn   c           	     ó  ‡ — ddl m} t        ‰ t        «      rÇt        j
                  r·t        j                  j                  ‰ j                  «       t        j                  «      r{t        t        j                  t        j                  j                  j                   j"                  «      rt%        t        j                  dd«      �t'        t        j                  d«      sy‰ j(                  t        j                  j*                  z  ‰ j,                  j.                  z  }dˆ fd„}‰ j1                  «       D �]Q  }|j2                  }|€J ‚|j5                  «       rr|j7                  «       sb|j9                  «       sR|j;                  «       t        j                  j<                  v s(t        |j?                  «       t@        jB                  «      rŒ—‰ jD                  jF                  D �]¡  }|jH                  ‰ j,                  jJ                  v r$‰ j,                  jJ                  |jH                     }n/‰ j,                  jL                  jO                  |jH                  «      }|sŒ|t        j                  jP                  jS                  |‰ «      sŒ§t        |jT                  tV        «      rŒÂ|jX                  €J ‚|jX                  D �cg c]   }|j2                  j;                  «       |vr|‘Œ" }	}‰ j,                  j[                  |jH                  ‰ «      }
|
r�Œ/t]        |	«      dk(  s�Œ?|	d   j^                  s�ŒP|	d   j2                  ‰ u s�Œc|j2                  €�Œqt        |j2                  j?                  «       t@        j`                  t@        jb                  t@        jd                  t@        jB                  f«      r�ŒÓ|jT                  rft        |jT                  j2                  t@        jf                  t@        jh                  f«      r(t]        |j2                  j7                  «       «      dkD  r�ŒE ||j2                  |j2                  «      s�Œd ||«      s�Œnt        j                  jj                  jm                  |j;                  «       |j;                  «       «       t        t        j                  t        j                  j                  j                   j"                  «      rnt        j                  jn                  jq                  |j;                  «       «       t        j                  jn                  jq                  |j;                  «       «       |j;                  «       t        j                  jr                  |j;                  «       <    �ŒQ �ŒT yc c}w )	z~
        Decide if there should be inplace updates for the node
        and record the decision in the active kernel.
        r   )Úcan_match_buffer_sizeÚ	mutationsNr4  c                óÂ  •‡— | j                   j                  ‰«      }| j                  «       Št        «       }| j                  D ]™  }|j
                  }t        |t        «      sŒ |j                  «       | j                   j                  vs| j                   j                  |«      |urŒd|ˆfd„|j                  j                  «       D «       z  }t        |«      dkD  sŒ™ y y)Nc              3  ó@   •K  — | ]  }|j                   ‰k(  r|–— Œ y ­wrw   r³  )r’   Úor  s     €rl   r”   z^BaseSchedulerNode.decide_inplace_update.<locals>.single_index_in_fused_node.<locals>.<genexpr>m  s%   øè ø€ ò àØ—v‘v Ò)ô ñùó   ƒr   FT)r  Úget_fused_noder  r   r  r�   rŒ   r_   rù  ró  r°   r¶  r©   )Úbuf_to_be_inplacedÚ
fused_noderß  r.  Ú	user_noder  rk   s        @€rl   Úsingle_index_in_fused_nodezKBaseSchedulerNode.decide_inplace_update.<locals>.single_index_in_fused_nodeU  sÝ   ù€ ð
 ,×5Ñ5×DÑDÀTÓJˆJØ)×2Ñ2Ó4ˆHô %/£LˆDØ*×0Ñ0ò !�Ø ŸI™I�	Ü! )Ô->Ô?Øð ×,Ñ,Ó.Ø-×7Ñ7×JÑJñKà)×3Ñ3×BÑBÀ9ÓMØ%ñ&ð àó à&×2Ñ2×CÑCÓEôñ �ô
 �t“9˜q“=Ù ð'!ð* rn   r   )r3  r  r†   rv   ):Úcodegen.wrapperr,  rŒ   r�   r(   Úinplace_buffersrZ   rœ   Úhas_featurerÞ   r.   ÚINPLACE_BUFFERSr=  rŸ  r   ÚcodegenÚsimdÚ
SIMDKernelr�  r<  rß   rç  r  Úcompleted_operationsrz  r�   r5  r6  r7  r  Úremoved_buffersr8  r+   r9  r°   r±   r²   r?  r@  rÈ  r:  Ú	can_reuser  ÚNopKernelSchedulerNoder  Úhas_cross_stream_hazardr©   r(  r?   r>   ÚMutationLayoutSHOULDREMOVEÚFallbackKernelr=   r4  Úmake_inplacer-  rÜ  r>  )rk   r,  Úinconsequential_nodesr6  rº   Úbuf_nodeÚreadÚ	input_bufÚxÚremaining_usesrB  s   `          rl   Údecide_inplace_updatez'BaseSchedulerNode.decide_inplace_update:  sô  ø€ õ
 	;ô �tœ]Ô+Ü×&Ò&Ü—‘×#Ñ# D§O¡OÓ$5´~×7UÑ7UÔVäœqŸx™x¬¯©×)@Ñ)@×)EÑ)E×)PÑ)PÔQÜœ1Ÿ8™8 [°$Ó7ÐCô œŸ™ &Ô)àð �N‰NÜ�g‰g×(Ñ(ñ)à�n‰n×1Ñ1ñ2ð 	õ 	ðD ×#Ñ#Ó%ó L	ˆCØ—x‘xˆHØÐ'Ð'Ð'à×,Ñ,Ô.Ø×8Ñ8Ô:Ø×.Ñ.Ô0Ø—<‘<“>¤Q§W¡W×%<Ñ%<Ñ<ô ˜h×6Ñ6Ó8¼"×:MÑ:MÔNàà×(Ñ(×.Ñ.ó >�à—9‘9 §¡× EÑ EÑEØ $§¡× EÑ EÀdÇiÁiÑ P‘Ià $§¡× :Ñ :× >Ñ >¸t¿y¹yÓ I�Iò ÜŸ™×,Ñ,×6Ñ6°yÀ$ÕGÜ& y×'<Ñ'<Ô>TÕUà$Ÿ?™?Ð6Ð6Ð6ð "+§¡ö&àØŸ6™6Ÿ?™?Ó,Ð4IÑIò ð&�Nð &ð
 /3¯n©n×.TÑ.TØŸ	™	 4ó/Ð+ó
 4Ü Ó/°1Ô4Ø*¨1Ñ-×9Ô9Ø*¨1Ñ-×2Ñ2°dÓ:Ø%ŸN™NÒ6Ü *Ø%ŸN™N×:Ñ:Ó<ä "§¡Ü "× 4Ñ 4Ü "× =Ñ =Ü "× 3Ñ 3ð	ö!ð &×1Ò1Ü *Ø )× 5Ñ 5× :Ñ :Ü!#×!2Ñ!2´B·N±NÐ Cô!ô !$ I§N¡N×$OÑ$OÓ$QÓ RÐUVÔ Vá1°)·.±.À#Ç(Á(ÖKÙ6°yÖAô
 Ÿ™Ÿ™×2Ñ2°9×3EÑ3EÓ3GÈÏÉËÔXä%ÜŸH™H¤e§o¡o×&=Ñ&=×&BÑ&B×&MÑ&Môô ŸH™H×.Ñ.×2Ñ2°9×3EÑ3EÓ3GÔHÜŸH™H×.Ñ.×2Ñ2°3·<±<³>ÔBð &×.Ñ.Ó0ô Ÿ™×7Ñ7¸¿¹»ÑGò ò}>ñL	ùò6&s   Ê?%V
c                óR  — t         j                  sy |r| j                  ry | j                  €J ‚| j                  j	                  «       }g }|D �]0  }|j
                  dk(  rŒ|j                  d«       |j                  d«       d|j
                  › d|j                  › �}d|j                  v r|d|j                  d   › �z   }|j                  |«       d|j                  v sŒ•|j                  d   › }|j                  d	d
¬«      d   }|j                  d|j                  dd«      j                  dd«      j                  dd«      j                  dd«      z   «       |j                  d«       |j                  d«       �Œ3 t        |«      dk(  ry |j                  |«       d| _        y )Nr{  r—  z#pragma CMT ORIGIN:z#pragma CMT ú Úseq_nrz seq_nr:Ústack_traceú|r   )Úmaxsplitr÷   ú{z{{ú}z}}r†  ú\z\\z#pragma CMT END ORIGINr   T)r(   Úcomment_originrr  r�   Úget_originsr  rÀ   ÚtargetÚmetaÚrsplitÚreplacer©   Ú
writelines)	rk   ÚbufferÚ	only_onceÚoriginsÚ	out_linesr0  Úop_info_strrP  Ústack_trace_last_lines	            rl   Úcodegen_originating_infoz*BaseSchedulerNode.codegen_originating_infoÅ  s™  € ô ×$Ò$Øá˜ŸšØØ�y‰yÐ$Ð$Ð$Ø—)‘)×'Ñ'Ó)ˆØˆ	àó 	%ˆAØ�t‰t�xÒàà×Ñ˜RÔ à×ÑÐ2Ô3Ø(¨¯©¨¨a°·±¨zÐ:ˆKØ˜1Ÿ6™6Ñ!Ø)¨h°q·v±v¸hÑ7GÐ6HÐ,IÑI�Ø×Ñ˜[Ô)Ø §¡Ò&Ø!"§¡¨Ñ!6Ð 7�Ø(3×(:Ñ(:¸3ÈÐ(:Ó(KÈBÑ(OÐ%Ø× Ñ Ø"Ø+×3Ñ3°C¸Ó>ß‘W˜S $Ó'ß‘W˜T 4Ó(ß‘WØ˜fóñ	ôð × Ñ Ð!9Ô:Ø× Ñ  Ö$ð3	%ô6 ˆy‹>˜QÒØð 	×Ñ˜)Ô$Øˆ�rn   c                ó(   — | j                  dd¬«      S )NT©Úinclude_readsÚinclude_writes©Ú!get_read_write_buffers_sizes_implrj   s    rl   Úget_read_write_buffers_sizesz.BaseSchedulerNode.get_read_write_buffers_sizesô  s   € à×5Ñ5Ø¨tð 6ó 
ð 	
rn   c                ó(   — | j                  dd¬«      S )NTFre  rh  rj   s    rl   Úget_read_buffer_sizesz'BaseSchedulerNode.get_read_buffer_sizesú  s   € à×5Ñ5Ø¨uð 6ó 
ð 	
rn   c                ó(   — | j                  dd¬«      S )NFTre  rh  rj   s    rl   Úget_write_buffer_sizesz(BaseSchedulerNode.get_write_buffer_sizes   s   € à×5Ñ5Ø°ð 6ó 
ð 	
rn   c                óZ   — t        | j                  ||¬«      j                  «       d¬«      S )Nre  r   )Ústart)rÙ   Úget_read_write_buffer_accessesr¹   )rk   rf  rg  s      rl   ri  z3BaseSchedulerNode.get_read_write_buffers_sizes_impl  s3   € ô Ø×/Ñ/Ø+¸Nð 0ó ç‰f‹hØô	
ð 	
rn   c                ó  ‡ ‡‡‡‡‡— t        ‰ t        «      ri S t        ‰ t        «      rt        ‰ j                  t        «      ri S t        ‰ t        «      r`t        ‰ j                  t
        j                  «      r<‰ j                  j                  t        j                  j                  j                  u ri S dd„Št        ‰ t        «      r@ ‰t        ‰ j                  «       d   «      t        ‰ j                  «       d   «      z  «      Šnt        d«      Št!        j"                  t$        «      }|r9‰ j&                  j(                  D ]   }||j*                     j-                  |«       Œ" |r9‰ j&                  j.                  D ]   }||j*                     j-                  |«       Œ" |r&t1        d„ ‰ j&                  j(                  D «       «      n	t1        «       }|r&t1        d„ ‰ j&                  j.                  D «       «      n	t1        «       }dˆ fd„Št        ‰ t2        «      rt1        ˆˆ fd„|D «       «      }||z
  }||z
  }i }||z  D ]º  }	t5        ˆfd	„||	   D «       «      Š|	t6        j8                  j:                  v rt6        j8                  j:                  |	   }
n;|	t6        j8                  j<                  v rt6        j8                  j<                  |	   }
nŒ�	 	 	 	 dˆˆˆ ˆfd
„Š ‰|
«      }|	|vr|||	<   Œ®||	xx   |z  cc<   Œ¼ |S )az  
        Counting the number of bytes accessed for a kernel is
        surprisingly tricky. In particular, there is a differentiation
        between 'theoretical' memory accesses and practical memory
        accesses. For example, a layernorm kernel may actually access an
        input 3 times, but in theory, it only needs to access its input
        once (and may be optimized to do so through say, persistent
        reductions)

        Another example is that even though a buffer is passed in, we may
        not access the entire buffer. This may occur if we are accessing
        a slice of the buffer. Another tricky case is for indirect
        indexing, where the amount of bytes accessed depends on the
        values of the input.

        What this function aims to compute is the memory accesses for
        worst-case inputs, best-case optimization. What this means is
        that for each buffer we compute the amount of potential accesses in two ways and take the minimum.

        1. Numel in ranges multiplied by number of deps the buffer has
        2. The buffer size

        Returns memory accesses per buffer.
        c                óX   — t         j                  j                  j                  | d¬«      S )Nr   rÉ   )rZ   rœ   r�   rË   )Úss    rl   Útry_size_hintzGBaseSchedulerNode.get_read_write_buffer_accesses.<locals>.try_size_hint:  s"   € Ü—7‘7×#Ñ#×5Ñ5°aÀ!Ð5ÓDÐDrn   r   r   ç    eÍÍAc              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   zCBaseSchedulerNode.get_read_write_buffer_accesses.<locals>.<genexpr>O  s   è ø€ ÒB C�s—x•xÑBùrµ  c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   zCBaseSchedulerNode.get_read_write_buffer_accesses.<locals>.<genexpr>T  ó   è ø€ ÒC C�s—x•xÑCùrµ  c                ó¢   •— ‰j                   j                  |    j                  }t        d„ |D «       «      }t	        |t        |«      z
  «      dkD  S )Nc              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r—   )r’   r.  s     rl   r”   z\BaseSchedulerNode.get_read_write_buffer_accesses.<locals>.is_materialized.<locals>.<genexpr>[  s   è ø€ Ò!>° $§)¥)Ñ!>ùrµ  r   )r  r@  r  r   r©   )rº   r·   r  Úbuf_usesrk   s       €rl   Úis_materializedzIBaseSchedulerNode.get_read_write_buffer_accesses.<locals>.is_materializedY  sG   ø€ Ø—N‘N×.Ñ.¨sÑ3×9Ñ9ˆEÜ!Ñ!>¸Ô!>Ó>ˆHÜ�x¤*¨VÓ"4Ñ4Ó5¸Ñ9Ð9rn   c              3  óJ   •K  — | ]  } ‰|‰j                   «      rŒ|–— Œ y ­wrw   ©r·   )r’   r¼   r}  rk   s     €€rl   r”   zCBaseSchedulerNode.get_read_write_buffer_accesses.<locals>.<genexpr>_  s#   øè ø€ ò )Ø©_¸SÀ$Ç+Á+Õ-N”ñ)ùs   ƒ#œ#c              3  ó"   •K  — | ]  }‰–— Œ y ­wrw   r~   )r’   r¼   Ú
node_numels     €rl   r”   zCBaseSchedulerNode.get_read_write_buffer_accesses.<locals>.<genexpr>h  s   øè ø€ Ò$R°C¤ZÑ$Rùs   ƒc                óB  •— | syt        | t        j                  «      r| j                  «       S t        | j                  t
        «      rÍ‰j                  j                  | j                  «          j                  }d}|D ]“  }t        |j                  t        «      rŒt        |j                  t        «      sJ ‚t        |j                  j                  t        «      r5|j                  j                  «       D ]  }| ‰|j                  «      z  }Œ Œ“ y |S t        | j                  t        j                  «      r"t!        ˆfd„| j#                  «       D «       «      S  ‰	t%        | j'                  «       «      «      }t)        | j+                  «       «      t-        ‰|«      z  S )Nr   c              3  óh   •K  — | ])  } ‰t         j                  j                  |«      «      –— Œ+ y ­wrw   )rZ   rœ   Ú
get_buffer)r’   Úmut_nameÚget_buf_bytess     €rl   r”   zZBaseSchedulerNode.get_read_write_buffer_accesses.<locals>.get_buf_bytes.<locals>.<genexpr>Œ  s-   øè ø€ ò à$ñ &¤a§g¡g×&8Ñ&8¸Ó&B×Cñùs   ƒ/2)rŒ   r+   ÚTorchBindObjectr†  r'  r>   r  r@  r  r  r�   rG  r_   r=   rz  r?   rÙ   r7  rY   r¢  rN   Ú	get_dtypeÚmin)
rº   r  Útotr.  Ú	sched_bufÚ	buf_elemsÚbuf_accessed_elemsr†  rk   ru  s
         €€€€rl   r†  zGBaseSchedulerNode.get_read_write_buffer_accesses.<locals>.get_buf_bytesq  sT  ø€ ñ Øä˜c¤2×#5Ñ#5Ô6Ø×,Ñ,Ó.Ð.Ü §
¡
Ô,=Ô>ð !ŸN™N×6Ñ6°s·|±|³~ÑF×LÑL�EØ�CØ %ò %˜Ü% d§i¡i´Ô<Ø$Ü)¨$¯)©)Ô5FÔGÐGÐGÜ% d§i¡i§n¡n´kÔBØ-1¯Y©Y×-BÑ-BÓ-Dò E 	Ø #¡}°Y·^±^Ó'DÑ D¡ñEñ $%ð%ð �JÜ §
¡
¬B¯M©MÔ:Üó à(+×(>Ñ(>Ó(@ôó ð ñ
 !.¬m¸C¿L¹L»NÓ.KÓ L�IÜ)¨#¯-©-«/Ó:¼SØ*¨Ió>ñ ð rn   )rt  z
sympy.Exprr†   r  )rº   r  r·   úSequence[BaseSchedulerNode]r†   rv   )rº   z4ir.Buffer | ir.TensorBox | ir.TorchBindObject | Noner†   r  )rŒ   rA  ÚExternKernelSchedulerNoder�   r=   r+   rD  Úop_overloadrŸ  Ú_primsÚ	rng_primsÚgraphsafe_run_with_rng_stater�   rY   Ú
get_rangesr  Úcollectionsr   rÿ   r°   r±   r²   rÀ   r�  r   rµ   rÙ   rZ   rœ   rÝ  Úgraph_inputs)rk   rf  rg  Úbuf_accessesr¼   r±   r�  r?  Úbuf_byte_accessesr  rº   Ú	buf_bytesr�  r†  r}  r�  ru  s   `           @@@@@rl   rq  z0BaseSchedulerNode.get_read_write_buffer_accesses  s»  ý€ ô6 �dÔ2Ô3ØˆIÜ�dÔ5Ô6¼:Ø�I‰I”{ô<
ð ˆIä�tÔ6Ô7Ü˜4Ÿ9™9¤b×&7Ñ&7Ô8Ø—	‘	×%Ñ%Ü�|‰|×%Ñ%×BÑBñCð ˆIó	Eô �dœMÔ*Ù&Ü˜dŸo™oÓ/°Ñ2Ó3Ü §¡Ó 1°!Ñ 4Ó5ñ6ó‰Jô
 ˜S›ˆJÜ"×.Ñ.¬tÓ4ˆáØ×'Ñ'×-Ñ-ò 3�Ø˜SŸX™XÑ&×-Ñ-¨cÕ2ð3ñ Ø×'Ñ'×.Ñ.ò 3�Ø˜SŸX™XÑ&×-Ñ-¨cÕ2ð3ñ
 ô ÑB¨4×+;Ñ+;×+AÑ+AÔBÔBä“ð 	ñ ô ÑC¨4×+;Ñ+;×+BÑ+BÔCÔCä“ð 	õ	:ô
 �dÔ.Ô/Ü(ô )Ø%ô)ó ˆOð ˜oÑ-ˆFØ˜OÑ+ˆEà,.Ðà ™ò 3	9ˆHÜ!$Ó$R¸<ÈÑ;QÔ$RÓ!RÐàœ1Ÿ7™7×1Ñ1Ñ1Ü—g‘g×,Ñ,¨XÑ6‘ØœQŸW™W×1Ñ1Ñ1Ü—g‘g×*Ñ*¨8Ñ4‘àð#ØIð#à÷#ð #ñJ & cÓ*ˆIØÐ0Ñ0Ø.7Ð! (Ò+à! (Ó+¨yÑ8Ô+ðg3	9ðj !Ð rn   c                óT  — | j                   €y | j                   j                  «       }|€y t        |«      }|€y t        |t        j
                  «      r|j                   j                  }t        j                  j                  j                  |d¬«      }t        d   dxx   |z  cc<   |S )Nr   rÉ   ÚinductorÚ
flop_count)r�   Úget_origin_noder9   rŒ   rŸ  ÚSymIntÚexprrZ   rœ   r�   rË   r   )rk   Úfx_nodeÚflopsÚresolved_flopss       rl   Úestimate_flopsz BaseSchedulerNode.estimate_flopsž  s�   € à�9‰9ÐØØ—)‘)×+Ñ+Ó-ˆØˆ?Øä˜wÓ'ˆØˆ=Øä�eœUŸ\™\Ô*Ø—J‘J—O‘OˆEäŸ™×)Ñ)×;Ñ;¸EÈAÐ;ÓNˆÜ�Ñ˜\Ó*¨nÑ<Ó*ØÐrn   c                óR   — | j                   �| j                   S | j                  «       S rw   )rn  Ú_get_estimated_runtimerj   s    rl   Úget_estimated_runtimez'BaseSchedulerNode.get_estimated_runtime±  s)   € Ø×*Ñ*Ð6Ø×2Ñ2Ð2à×*Ñ*Ó,Ð,rn   c                óæ  — | j                  «       d   j                  «       d   }|j                  j                  «       }t	        t        |«      «      syt        | j                  «      r¿t        | j                  t        j                  «      sJ ‚	 t        j                  rst        | «      }t        «       }|j                  |«      }|�t        |t        «      sJ ‚|S t!        | «      }|€t#        | j                  «      }|j%                  ||¬«       |S t#        | j                  «      S t/        | j                  «      ryt1        | «      }|�|S |j                  j3                  «       }		 t5        «       }
t7        |	«      dz  }|
dk  rt9        d|
› �«      ‚|dk  rt9        d|› �«      ‚	 | j=                  «       }|dk(  s|€| j?                  «       |
z  }|dz  }|S d}| j?                  «       }|€dn|}||z  |z  d	z  }||
z  }tA        ||«      }|dz  }|S # t&        $ r}t(        j+                  |«       Y d}~yd}~wt,        $ r}t(        j+                  |«       Y d}~yd}~ww xY w# t:        $ r Y yw xY w)
zC
        Returns estimated op runtime in milliseconds (ms)
        r   N©Úvaluel    J)£z-gpu_memory_bandwidth cannot be <= 0, but got z"gpu_flops cannot be <= 0, but got g    €„.Ag      ð?rv  )!r–   rz  r�   r8  rU   r;   rS   rŒ   r+   ÚIRNoder)   Ú,runtime_estimations_use_nccl_lib_estimationsÚ)get_estimate_runtime_cache_key_from_snodeÚget_estimate_runtime_cacheÚlookupÚfloatr2   r1   Ú	set_valueÚ
ValueErrorr“  r©  Ú	TypeErrorrX   Ú maybe_estimate_runtime_benchmarkÚmaybe_get_dtyperO   rM   ÚAssertionErrorr’  r£  rj  Úmax)rk   rº   r'  Ú	cache_keyÚcacheÚ	cache_valÚmsr  ÚretÚdtypeÚgpu_memory_bandwidthÚ	gpu_flopsÚ	flops_estÚnsÚfactorÚcounted_bytesÚcompute_timeÚtransfer_times                     rl   r¥  z(BaseSchedulerNode._get_estimated_runtime·  sw  € ð
 �n‰nÓ˜qÑ!×-Ñ-Ó/°Ñ2ˆØ—‘×)Ñ)Ó+ˆÜ”o fÓ-Ô.àô ˜Ÿ™Ô#Ü˜dŸi™i¬¯©Ô3Ð3Ð3ðÜ×LÒLÜ IÈ$Ó O�IÜ6Ó8�EØ %§¡¨YÓ 7�IØ Ð,Ü)¨)´UÔ;Ð;Ð;Ø(Ð(äHÈÓN�BØ�zä=¸d¿i¹iÓH˜à—O‘O I°R�OÔ8Ø�IÜ7¸¿	¹	ÓBÐBô �T—Y‘YÔð
 ä.¨tÓ4ˆØˆ?ØˆJà—‘×(Ñ(Ó*ˆð	Ü#4Ó#6Ð Ü)¨%Ó0°6Ñ9ˆIð $ qÒ(Ü$ØCÐDXÐCYÐZóð ð ˜AŠ~Ü$Ð'IÈ)ÈÐ%UÓVÐVð ð
 ×'Ñ'Ó)ˆ	à˜Š>˜YÐ.à×2Ñ2Ó4Ð7KÑKˆBØ�c‘ˆBØˆIð ˆØ×9Ñ9Ó;ˆØ*Ð2™¸ˆØ Ñ*¨YÑ6¸#Ñ=ˆØ%Ð(<Ñ<ˆô �˜}Ó-ˆØ�#‰XˆØˆ	øôo ò ô —‘˜”ÜûÜò ä—‘˜”Üûðûô: ò 	Ùð	úsC   ÂAH Ã6H ÄH Å(>I$ È	I!ÈH6È6I!ÉIÉI!É$	I0É/I0c                 ó   — y rw   r~   rj   s    rl   Úget_template_nodez#BaseSchedulerNode.get_template_node  ó   € Ørn   c                ó.   — | j                  «       }|€J ‚|S rw   ©rÆ  )rk   Útemplates     rl   Úget_template_node_or_throwz,BaseSchedulerNode.get_template_node_or_throw  s!   € Ø×)Ñ)Ó+ˆØÐ#Ð#Ð#Øˆrn   c                óf   — t        d„ t        | «      D «       «      }| d| }| |   }| |dz   d }|||fS )zQ
        For the list of nodes, get the prologue, template, and epilogue
        c              3  óH   K  — | ]  \  }}|j                  «       sŒ|–— Œ y ­wrw   ©r   )r’   Úir  s      rl   r”   zCBaseSchedulerNode.get_prologue_template_epilogue.<locals>.<genexpr>  s   è ø€ ÒP¡D A qÀÇÁÅœaÑPùs   ‚"›"Nr   )ÚnextÚ	enumerate)ÚnodesÚtemplate_indexÚprologueÚtemplate_nodeÚepilogues        rl   Úget_prologue_template_epiloguez0BaseSchedulerNode.get_prologue_template_epilogue  sN   € ô ÑP¬I°eÓ,<ÔPÓPˆà˜˜.Ð)ˆØ˜nÑ-ˆØ˜¨!Ñ+Ð-Ð.ˆØ˜¨Ð0Ð0rn   )r  r  r†   rU  )r�   úir.Operationr†   rU  rR  )r†   r  rT  ©r­  r4   r®  r4   r†   rv   ©r¹  rf  r†   rU  )r¼   r3   r†   rU  rV  )rÅ  ro  r†   rU  ©rÍ  r_  rÊ  rf  r†   rU  ©r†   r_  ©ró  údict[str, BaseSchedulerNode]r†   rU  ©r†   rŽ  )r†   zSequence[SchedulerBuffer])r  r  r†   r  rX  ©r'  zdependencies.Depr†   rv   ©T)r]  rR   r^  rv   r†   rU  rS  )rf  rv   rg  rv   r†   r  )rf  rv   rg  rv   r†   zdict[str, int]©r†   ú
int | None©r†   r¯  ©r†   zir.TemplateBuffer | None)r†   zir.TemplateBuffer)rÒ  úlist[BaseSchedulerNode]r†   zJtuple[list[BaseSchedulerNode], BaseSchedulerNode, list[BaseSchedulerNode]])>ry   rz   r{   r|   r�   rn  rr  rx  r|  r€  r/  r‘  rš  r§  rª  r¯  rº  r¾  rÂ  r·  rÏ  rÑ  r¿   rÌ  rÄ  rð  rô  r  rù  rI   rà   r  r	  r-   r–   rz  r  rÞ   r  rU   rŽ   r  r  r   r"  r$  r(  r*  rL  rc  rj  rl  rn  ri  rq  r£  r¦  r¥  rÆ  rË  r  r×  r~   rn   rl   r_   r_     sÁ  … ØÓØBÓBØÓð ÓØÓØƒNØƒNØ'Ó'Ø$Ó$Ø $€DÐ
Ó$Ø"Ó"Ø/Ó/Ø/3Ð Ó3Ø(Ó(Ø'Ó'Ø€GˆTÓó
ó#ó4Bó*ó6ó+ó#ó
ðØ!ðØ.7ðà	óó
Mó>ó
ó
ð
=Ø#2ð=ØHVð=à	ó=óó
óó6
óGð TØ">ðTà	óTó
.óð òHó ðHð òBó ðBð ò
ó ð
ð ò
ó ð
óóó.ó&ó;ó:óóóóóóóóóIðX 9=ð-Ø$ð-Ø15ð-à	ó-ð^ ò
ó ð
ð
 ò
ó ð
ð
 ò
ó ð
ð

Ø!ð
Ø37ð
à	ó
ðL!Ø!ðL!Ø37ðL!à	óL!ð\ òó ðó$-ð òUó ðUónóð
 ð1Ø&ð1à	Sò1ó ñ1rn   c                 óR   — t         j                  j                  j                  «       S rw   )rŸ  r   Ú	codecacheÚ
LocalCacher~   rn   rl   r­  r­  &  s   € ä�?‰?×$Ñ$×/Ñ/Ó1Ð1rn   c                óœ  ‡— t        | j                  dd«      }| j                  j                  }| j                  j                  g |¢| j                  j                  ¢| j                  j
                  «      }| j                  j
                  }t        j                  ||f«      \  }}dd„Št        |ft        ˆfd„|D «       «      z   «      }|S )NÚpython_kernel_namer—  c                ó�   — t        | t        j                  «      xr+ t        | t        j                  t        j                  f«       S rw   )rŒ   r+   rª  ÚGeneratorStateÚOpaqueObjectState©rJ  s    rl   Ú_is_tensor_irz@get_estimate_runtime_cache_key_from_snode.<locals>._is_tensor_ir5  s<   € Ü˜!œRŸY™YÓ'ò 
´
Ø”×!Ñ!¤2×#7Ñ#7Ð8ó1
ð -
ð 	
rn   c              3  ód   •K  — | ]'  } ‰|«      rt        |j                  «       «      nd –— Œ) y ­wrw   )rª   r¢  )r’   Úarð  s     €rl   r”   z<get_estimate_runtime_cache_key_from_snode.<locals>.<genexpr><  s(   øè ø€ ÒUÀa¡}°QÔ'7”�a—j‘j“lÔ#¸TÓAÑUùs   ƒ-0rV  )
r�  r�   ÚinputsÚfill_non_provided_argsÚconstant_argsru  ÚpytreeÚtree_flattenr  rª   )Úsnoderë  r4  ru  Ú	flat_argsÚflat_args_pytree_specr·  rð  s          @rl   r¬  r¬  +  s¼   ø€ Ü  §¡Ð-AÀ2ÓFÐØ�:‰:×Ñ€DØ�:‰:×,Ñ,Ø*ˆ$Ð*�—‘×)Ñ)Ð*Ø�
‰
×Ñó€Dð �Z‰Z×Ñ€FÜ'-×':Ñ':¸DÀ&¸>Ó'JÑ$€IÐ$ó
ô
 Ø	ÐÜ
ÓUÈ9ÔUÓ
Uñ	Vó€Ið Ðrn   c                óˆ  — t        | t        «      sy t        j                  j                  j
                  t        j                  j                  j                  t        j                  j                  j                  dœ}t        | j                  dd«      }||vry t        | j                  t        j                  «      sy ||   S )N)zextern_kernels.mmzextern_kernels.bmmzextern_kernels.addmmrë  r—  )rŒ   r�  rŸ  ÚopsÚatenÚmmÚbmmÚaddmmr�  r�   r+   ÚExternKernel)rø  Úmms_fnsrë  s      rl   Ú_get_mm_like_fnr  A  sŒ   € Ü�eÔ6Ô7Øä"ŸY™YŸ^™^×.Ñ.Ü#Ÿi™iŸn™n×0Ñ0Ü %§	¡	§¡× 4Ñ 4ñ€Gô
 ! §¡Ð-AÀ2ÓFÐØ Ñ(ØÜ�e—j‘j¤"§/¡/Ô2ØØÐ%Ñ&Ð&rn   c                óZ  ‡ ‡— d }d }t         j                  rt        ‰ «      }|€y |}ˆ ˆfd„}ny t        ‰ «      }t	        «       }|j                  |«      }|�t        |t        «      sJ ‚|S ddlm	Š  |«       \  }}ddl
m}	 |	j                  |||ddd¬«      }
|j                  ||
¬	«       |
S )
Nc                 ó   •—  ‰‰ «      S rw   r~   )rø  Úsnode_args_kwargss   €€rl   rv  z2maybe_estimate_runtime_benchmark.<locals>.<lambda>Z  s   ø€ Ñ!2°5Ó!9€ rn   r   )r  r   )Úbenchmarkeré   é
   )Úmemory_warmup_itersÚbenchmark_itersÚmax_benchmark_durationr¨  )r(   Ú!runtime_estimations_mms_benchmarkr  r¬  r­  r®  rŒ   r¯  Úutilsr  Ú$torch._inductor.runtime.benchmarkingr  Ú	benchmarkr°  )rø  Úbench_fnÚargs_kwargs_fnÚmm_fnr·  r¸  r¹  r4  ru  r  rº  r  s   `          @rl   r³  r³  Q  sÁ   ù€ Ø€HØ€NÜ×/Ò/Ü Ó&ˆØˆ=ØØˆä9‰àä9¸%Ó@€IÜ&Ó(€EØ—‘˜YÓ'€IØÐÜ˜)¤UÔ+Ð+Ð+ØÐå(á!Ó#�L€Dˆ&Ý@à	×	Ñ	ØØØØØØ!ð 
ó 
€Bð 
‡O�O�I R€OÔ(Ø€Irn   T)Úslotsc                  óN   — e Zd ZU ded<   ded<   ded<   ded<   dd„Zdd„Zdd	„Zy
)Ú	WhyNoFuser  Úname1Úname2Úreasonztuple[Any, ...]r4  c                óX   — |j                  «       | _        |j                  «       | _        y rw   )r  r  r  ©rk   r�   r‚   s      rl   rx  zWhyNoFuse.__init__~  s   € Ø—^‘^Ó%ˆŒ
Ø—^‘^Ó%ˆ�
rn   c                óJ   — || _         || _        t        j                  | «       y rw   )r  r4  Ú
fusion_logÚdebug)rk   r  r4  s      rl   Ú__call__zWhyNoFuse.__call__‚  s   € ØˆŒØˆŒ	Ü×Ñ˜Õrn   c                óp   — d| j                   › d| j                  › d�| j                  | j                  z  z   S )Nzcannot fuse z with r#  )r  r  r  r4  rj   s    rl   Ú__str__zWhyNoFuse.__str__‡  s6   € Ø˜dŸj™j˜\¨°·
±
¨|¸2Ð>Ø�K‰K˜$Ÿ)™)Ñ#ñ
ð 	
rn   N©r�   r_   r‚   r_   r†   rU  )r  r  r4  r   r†   rU  rR  )ry   rz   r{   r|   rx  r  r!  r~   rn   rl   r  r  w  s&   … àƒJØƒJØƒKØ
Óó&óô

rn   r  c                ó¼   — t        | t        t        f«      rt        | t        ¬«      } t        j                  | d¬«      }d|v rdt        j                  |d«      › �S |S )N©Úkeyé   )r+  r†  ú    )	rŒ   r   ÚsetÚsortedr  Úpprintr)  Útextwrapr+  )Úobjr-  s     rl   r)  r)  �  sR   € Ü�#œ
¤CÐ(Ô)ä�SœcÔ"ˆÜ�^‰^˜C¨Ô*€FØˆv�~Ø”H—O‘O F¨GÓ4Ð5Ð6Ð6Ø€Mrn   c                  ó0   — e Zd Zdd„Zdd„Zdd„Zd	d„ZeZy)
rG  c                ó&   — t        |g«      | _        y rw   rä  r½  s     rl   rx  zOutputNode.__init__˜  s   € Ü",¨c¨UÓ"3ˆÕrn   c                 ó   — yr¬  r~   rj   s    rl   rŽ   zOutputNode.is_reduction›  r  rn   c                 ó   — y)Nr~   r~   rj   s    rl   r6  z'OutputNode.get_inputs_that_alias_outputž  r˜  rn   c                 ó   — y)NÚOUTPUTr~   rj   s    rl   r  zOutputNode.get_name¡  s   € Ørn   N)r¼   r5   r†   rU  rV  rW  rR  )ry   rz   r{   rx  rŽ   r6  r  r€  r~   rn   rl   rG  rG  —  s   „ ó4óóóð �Hrn   rG  c                óÀ  ‡ ‡‡‡‡— t        j                  «       Š‰ j                  D ]N  }t        |t        «      rŒ‰|j
                     j                  «       }‰‰|   j                  «       xx   dz  cc<   ŒP dˆˆˆˆ fd„Št        ˆfd„‰ j                  D «       «      }|r?‰ j                  |z
  ‰ _        ‰ j                  ‰ j                  j                  |«      «       yy)am  
    Prunes weakdeps intended for mutation ordering
    on an upstream fused node if after fusion there is another dependency
    on the fused upstream node, making the weakdep redundant

    In essence this enforces an ordering on fusions. As fusions occur, weakdeps will
    be incrementally removed, enabling other fusions, ensuring they are fused in order.
    r   c                óò   •— t        | t        «      rf‰| j                     j                  «       }‰‰|   j	                  «          dkD  xr  ‰j
                  j                  | ‰|   ‰«      }‰|   ‰k(  }|xs |S y)Nr   F)rŒ   r6   r²   r  r  r  Úfusable_weak_dep)r¼   rè  Úis_redundantÚis_self_depr@  Úname_to_dep_countró  r�   s       €€€€rl   ré  z+_prune_redundant_deps.<locals>.should_prune»  s�   ø€ Ü�cœ7Ô#Ø! #§(¡(Ñ+×<Ñ<Ó>ˆGØ,Ø" 7Ñ+×4Ñ4Ó6ñàñò à—n‘n×5Ñ5ØÐ'¨Ñ0°$óð ð -¨WÑ5¸Ñ=ˆKØÒ. ;Ð.àrn   c              3  ó4   •K  — | ]  } ‰|«      sŒ|–— Œ y ­wrw   r~   rë  s     €rl   r”   z(_prune_redundant_deps.<locals>.<genexpr>Ë  s   øè ø€ ò Ø±,¸sÕ2CŒñùrì  Nrí  )r•  r   rq  rŒ   r6   r²   r  r  r   r·  r°   rî  )r�   ró  r@  r¼   rè  Údeps_to_pruner8  ré  s   ```   @@rl   rò  rò  §  sÐ   ü€ ô '2×&9Ñ&9Ó&;Ðà×&Ñ&ò KˆÜ˜#œwÕ'Ø! #§(¡(Ñ+×<Ñ<Ó>ˆGØÐ0°Ñ9×BÑBÓDÓEÈÑJÔEðK÷
ð ô  ó Ø×.Ñ.ôó €Mñ Ø"&×"9Ñ"9¸MÑ"IˆÔØ×Ñ˜T×-Ñ-×:Ñ:¸=ÓIÕJð rn   c                  óH   ‡ — e Zd Zdˆ fd„Zdd„Zd	d„Zd	d„Zd
d„Zdd„Zˆ xZ	S )r�  c                ón  •— t         ‰| �  |«       | j                  |«       | j                  |j	                  «       «       t        |t        j                  «      r[|j                  «       rJt        j                  |j                  d   j                  «      }d}|j                  «       }|||ff| _        y y y ©Nr   r   )Úsuperrx  r|  r·  Úget_read_writesrŒ   r+   ÚUserDefinedTritonKernelÚcan_fuse_epilogueÚmathrÚ   Úmutable_argsÚshapeÚget_device_or_errorr¡   )rk   r  r�   Únumelr£   r  Ú	__class__s         €rl   rx  z"ExternKernelSchedulerNode.__init__Õ  s›   ø€ Ü‰Ñ˜Ô#Ø×Ñ˜TÔ"Ø×Ñ˜T×1Ñ1Ó3Ô4ä�dœB×6Ñ6Ô7¸D×<RÑ<RÔ<TÜ—I‘I˜d×/Ñ/°Ñ2×8Ñ8Ó9ˆEØˆFØ×-Ñ-Ó/ˆFà  5¨& /Ð2ˆD�Jð =UÐ7rn   c                óV   — | j                  «       › dt        | j                  dd «      › �S )Nz.node.kernel = rë  )r  r�  r�   rj   s    rl   r‘  z)ExternKernelSchedulerNode.debug_str_extraá  s*   € Ø—-‘-“/Ð" /´'¸$¿)¹)ÐEYÐ[_Ó2`Ð1aÐbÐbrn   c                 ó   — y©NTr~   rj   s    rl   r"  z#ExternKernelSchedulerNode.is_externä  rÇ  rn   c                ó‚   — | j                   €J ‚t        | j                   d«      xr | j                   j                  «       S )Nr*  )r�   r<  r*  rj   s    rl   r*  z*ExternKernelSchedulerNode.has_side_effectsç  s6   € Ø�y‰yÐ$Ð$Ð$Ü�t—y‘yÐ"4Ó5ÒV¸$¿)¹)×:TÑ:TÓ:VÐVrn   c                óü   — t        | j                  t        j                  «      rU| j                  j	                  «       r;t        j                  | j                  j                  d   j                  «      }|gg fS g g fS rÅ   )	rŒ   r�   r+   r@  rA  rB  rÚ   rC  rD  )rk   rF  s     rl   r”  z$ExternKernelSchedulerNode.get_rangesë  s^   € ä�t—y‘y¤"×"<Ñ"<Ô=Ø—	‘	×+Ñ+Ô-ä—I‘I˜dŸi™i×4Ñ4°QÑ7×=Ñ=Ó>ˆEØ�G˜R�=Ð Ø�Bˆxˆrn   c                ó„   — t        | j                  t        j                  «      sJ ‚| j                  j	                  |«      S rw   )rŒ   r�   r+   r  r;  )rk   Úwrappers     rl   r;  z!ExternKernelSchedulerNode.codegenô  s/   € Ü˜$Ÿ)™)¤R§_¡_Ô5Ð5Ð5Ø�y‰y× Ñ  Ó)Ð)rn   ©r  r  r�   rØ  r†   rU  rR  rV  ©r†   úSequence[Sequence[sympy.Expr]]©rN  r   r†   rU  )
ry   rz   r{   rx  r‘  r"  r*  r”  r;  Ú__classcell__©rG  s   @rl   r�  r�  Ô  s#   ø„ õ
3ócóóWó÷*rn   r�  c                  ó    ‡ — e Zd Zdˆ fd„Zˆ xZS )rA  c                ó„   •— t         ‰| �  |«       | j                  |«       | j                  |j	                  «       «       y rw   )r>  rx  r|  r·  r?  ©rk   r  r�   rG  s      €rl   rx  zNopKernelSchedulerNode.__init__ú  s5   ø€ Ü‰Ñ˜Ô#Ø×Ñ˜TÔ"Ø×Ñ˜T×1Ñ1Ó3Õ4rn   rO  )ry   rz   r{   rx  rS  rT  s   @rl   rA  rA  ù  s   ø„ ÷5ñ 5rn   rA  c                  ó˜  ‡ — e Zd ZU dZded<   ded<   	 	 	 	 	 	 d!ˆ fd„Z	 	 d"	 	 	 	 	 d#d„Z	 	 d"	 	 	 	 	 d$d„Z	 	 	 	 	 	 d%d	„Zd&d
„Z	d'd„Z
d(d„Zd'd„Z	 	 	 	 	 	 d)d„Zd'd„Z	 	 	 	 	 	 d*d„Zd+d„Zd,d„Zd-d„Zd-d„Zd-d„Zd-d„Zd.d„Zd/d„Z	 	 	 	 d0d„Zd1d„Z	 d2	 	 	 d3d„Zed4d„«       Zed4d„«       Zd5d„Zed6d„«       Zed-ˆ fd „«       Z ˆ xZ!S )7r�   zu
    A SchedulerNode is a node for scheduling that encapsulates either
    a ComputedBuffer or a TemplateBuffer.
    z tuple[Sequence[sympy.Expr], ...]Ú_sizesr@   rù   c                óf   •— t         ‰| �  |«       | j                  |«       | j                  «        y rw   )r>  rx  r|  Ú_compute_attrsrW  s      €rl   rx  zSchedulerNode.__init__	  s,   ø€ ô
 	‰Ñ˜Ô#Ø×Ñ˜TÔ"Ø×ÑÕrn   c                óâ  — t        | j                  t        j                  t        j                  f«      sJ ‚| j                  j                  ||¬«      \  | _        }|| _        | j                  j                  «       }| j                  j                  |«      j                  }| || j                  «      f| _        t        j                   xs t        |j                   «       }t        | j                  t        j                  «      r,| j#                  | j                  j%                  |¬«      «       y | j#                  t'        j$                  | j                  g| j                  ¢­d|iŽ«       y )N©Úextra_indexing_constraintsÚrecompute_sizes_body_func)Ú	normalizer`  )rŒ   r�   r+   r   ÚTemplateBufferÚsimplify_and_reorderrY  rù   rE  r  Úget_backendÚgroup_fnr¡   r(   Úloop_ordering_after_fusionrU   r¶   r·  Úextract_read_writesr*   )rk   r^  r_  Úbodyr  rd  Úshould_normalizes          rl   r[  zSchedulerNode._compute_attrs  s7  € ô
 ˜$Ÿ)™)¤b×&7Ñ&7¼×9JÑ9JÐ%KÔLÐLÐLØ ŸI™I×:Ñ:Ø'AØ&?ð ;ó 
ÑˆŒ�Tð ˆŒ
à—‘×.Ñ.Ó0ˆØ—>‘>×-Ñ-¨fÓ5×>Ñ>ˆØ™h t§{¡{Ó3Ð4ˆŒ
ô  &×@Ñ@Ð@ò 
ÌØ�K‰KóI
ð E
Ðô �d—i‘i¤×!2Ñ!2Ô3Ø× Ñ Ø—	‘	×-Ñ-Ð8HÐ-ÓIõð × Ñ Ü×0Ñ0Ø—J‘JðØ!%§¡òØ8Hñõrn   c                ó  — t        d„ | j                  j                  D «       «      }| j                  ||¬«       |rD| j	                  | j                  j                  |«      j                  | j                  «      «       y y )Nc              3  óN   K  — | ]  }t        |t        t        f«      sŒ|–— Œ y ­wrw   ©rŒ   r6   r5   r´  s     rl   r”   z8SchedulerNode.recompute_size_and_body.<locals>.<genexpr>8  ó#   è ø€ ò 0
Ø´ZÀÄgÌwÐEWÕ5XŒCñ0
ùó   ‚%ž%r]  )r   r°   r±   r[  r·  r¼  r¸  rg  )rk   r^  r_  Ú	fake_depss       rl   Úrecompute_size_and_bodyz%SchedulerNode.recompute_size_and_body3  s   € ô
 &0ñ 0
Ø×+Ñ+×1Ñ1ô0
ó &
ˆ	ð 	×ÑØ'AØ&?ð 	ô 	
ñ Ø× Ñ Ø× Ñ ×*Ñ*¨9Ó5×<Ñ<¸T×=RÑ=RÓSõð rn   c                ó”  — t        d„ | j                  j                  D «       «      }| j                  t	        j
                  | j                  g| j                  ¢­d|iŽj                  |«      j                  | j                  «      «       | j                  j                  | «       |r!ddlm} |j                  j!                  «        y y )Nc              3  óN   K  — | ]  }t        |t        t        f«      sŒ|–— Œ y ­wrw   rk  r´  s     rl   r”   z5SchedulerNode.refresh_dependencies.<locals>.<genexpr>I  rl  rm  r`  r   ©ÚSIMDScheduling)r   r°   r±   r·  r*   rf  rù   rY  r¼  r¸  rg  Úpointwise_read_writesÚclear_cacheÚcodegen.simdrs  Úcandidate_tilingsÚcache_clear)rk   r`  Úneed_clear_tiling_cachern  rs  s        rl   Úrefresh_dependenciesz"SchedulerNode.refresh_dependenciesD  s±   € ô
 &0ñ 0
Ø×+Ñ+×1Ñ1ô0
ó &
ˆ	ð 	×ÑÜ×,Ñ,Ø—
‘
ðØ!Ÿ[™[òØ4=ñ÷ ‰Y�yÓ!ß‰V�D×)Ñ)Ó*ô	
ð 	×"Ñ"×.Ñ.¨tÔ4á"Ý4ð ×,Ñ,×8Ñ8Õ:ð #rn   c                ó    — | j                   j                  |«      | _         | j                   j                  | _        | j	                  dd¬«       y )NFT©r`  ry  )rù   Úreorder_iter_loopsÚsizesrY  rz  )rk   Ú	new_orders     rl   Úapply_new_loop_orderz"SchedulerNode.apply_new_loop_orderb  sA   € Ø—Z‘Z×2Ñ2Øó
ˆŒ
ð —j‘j×&Ñ&ˆŒà×!Ñ!¨EÈ4Ð!ÕPrn   c                ó¦  — | j                   j                  «       }t        | j                   j                  «      |z
  }t	        t        |«      «      }t	        t        |||z   «      «      }| j                  ||z   «       t        | j                  d   «      dk(  sJ ‚| j                  d   | j                  d   d   | j                  d   d   ff| _        y )Nr   r¨   r   )rù   Úget_original_num_rdimsr©   Ú	iter_varsrª   Úranger€  r¡   )rk   Ú	num_rdimsÚ
num_pwdimsÚpwdimsÚrdimss        rl   Úswap_pw_red_dimensionz#SchedulerNode.swap_pw_red_dimensionj  s²   € Ø—J‘J×5Ñ5Ó7ˆ	Ü˜Ÿ™×-Ñ-Ó.°Ñ:ˆ
Ü”u˜ZÓ(Ó)ˆÜ”e˜J¨
°YÑ(>Ó?Ó@ˆà×!Ñ! %¨&¡.Ô1Ü�4—:‘:˜a‘=Ó! QÒ&Ð&Ð&Ø—Z‘Z ‘] T§Z¡Z°¡]°1Ñ%5°t·z±zÀ!±}ÀQÑ7GÐ$HÐHˆ�
rn   c                óD   — | j                   j                  «       | _         | S rw   )rù   Úextract_pw_from_reductionrj   s    rl   r‹  z'SchedulerNode.extract_pw_from_reductiont  s   € Ø—Z‘Z×9Ñ9Ó;ˆŒ
Øˆrn   c                óú   — t         j                  | «      sy t        | j                  t        j
                  «      sJ ‚| j                  j                  «       5  | j                  «        d d d «       y # 1 sw Y   y xY wrw   )r‰   r˜   rŒ   r�   r+   r   Úwith_original_inner_fnr[  rj   s    rl   Úcancel_reduction_splitz$SchedulerNode.cancel_reduction_splitx  s^   € Ü ×3Ñ3°DÔ9ØÜ˜$Ÿ)™)¤R×%6Ñ%6Ô7Ð7Ð7Ø�Y‰Y×-Ñ-Ó/ñ 	"Ø×ÑÔ!÷	"÷ 	"ñ 	"ús   ÁA1Á1A:c                ó¾  — t        | j                  t        j                  t        j                  f«      sJ ‚| j
                  j                  ||«      | _        | j
                  j                  | _        | j                  j                  «       }| j                  j                  |«      j                  }| || j                  «      f| _        | j                  dd¬«       y )NTr|  )rŒ   r�   r+   r   ra  rù   Ú#expand_dimension_for_pointwise_noder~  rY  rE  r  rc  rd  r¡   rz  )rk   Ú	dimensionÚ	new_ranger  rd  s        rl   r�  z1SchedulerNode.expand_dimension_for_pointwise_node  s­   € ô ˜$Ÿ)™)¤b×&7Ñ&7¼×9JÑ9JÐ%KÔLÐLÐLà—Z‘Z×CÑCØ�yó
ˆŒ
ð —j‘j×&Ñ&ˆŒà—‘×.Ñ.Ó0ˆØ—>‘>×-Ñ-¨fÓ5×>Ñ>ˆØ™h t§{¡{Ó3Ð4ˆŒ
ð 	×!Ñ!¨DÈ$Ð!ÕOrn   c                óž   — | j                   j                  «       | _         | j                   j                  | _        | j	                  dd¬«       y )NTFr|  )rù   Úmerge_loopsr~  rY  rz  rj   s    rl   r”  zSchedulerNode.merge_loops�  s<   € Ø—Z‘Z×+Ñ+Ó-ˆŒ
Ø—j‘j×&Ñ&ˆŒð 	×!Ñ!¨DÈ%Ð!ÕPrn   c                óŠ  — d }| j                   d   }t        |«      |j                  cxk(  r|j                  k(  rn n|j                  |«      }|rPt        xj
                  dz  c_        t        j                  d| j                  «       |«       | j                  |«       yt        j                  d| j                  «       «       y)Nr   r   z"Reorder loops for %s with order %sTzEDon't reordering %s because we can not decide the suitable loop orderF)
rY  r©   Únum_varsÚdecide_loop_order_to_matchr,   Únum_loop_reorderingÚloop_ordering_logr  r  r€  )rk   r­  r®  r  Ú
self_sizess        rl   r¯  z'SchedulerNode.reorder_loops_by_dep_pairœ  s¤   € ð ˆ	Ø—[‘[ ‘^ˆ
Üˆz‹?˜h×/Ñ/ÔE°9×3EÑ3EÕEØ ×;Ñ;¸IÓFˆIáä×'Ò'¨1Ñ,Õ'Ü×#Ñ#Ø4°d·m±m³oÀyôð ×%Ñ% iÔ0Øä×#Ñ#ØWØ—‘“ôð rn   c                ó$  — | j                  «       }|› d| j                  d   › �|› d| j                  d   › �|› d| j                  › �g}| j                  j	                  «       D ]‚  }t        |t        «      rŒ|j                  }t        j                  j                  |«      }t        |t        j                  «      rŒZ|j                  |› dt        |j                  «      › �«       Œ„ t        | j                   t"        «      rR|j                  d|› d�«       |j                  t%        j&                  | j                   j)                  «       d	«      «       | j*                  €J ‚|j-                  | j/                  «       «       d
j1                  |«      S )Nz.group.device = r   z.group.iteration = r   z	.sizes = z
_layout = zclass z_loop_body:r'  r†  )r  r¡   rY  r°   r¶  rŒ   r6   r²   rZ   rœ   r„  r+   r‡  rÀ   r)  r'  rù   r@   r+  r+  r/  r�   rÞ  rš  Újoin)rk   r²   Úlinesr¼   r  rº   s         rl   r‘  zSchedulerNode.debug_str_extra³  sK  € Ø�}‰}‹ˆàˆfÐ$ T§Z¡Z°¡] OÐ4ØˆfÐ'¨¯
©
°1© Ð7Øˆf�I˜dŸk™k˜]Ð+ð
ˆð
 ×#Ñ#×4Ñ4Ó6ò 	OˆCÜ˜c¤7Õ+ØŸ8™8�Ü—g‘g×(Ñ(¨Ó2�Ü! #¤r×'9Ñ'9Õ:Ø—L‘L H :¨Z¼ÀÇ
Á
Ó8KÐ7LÐ!MÕNð	Oô �d—j‘j¤(Ô+Ø�L‰L˜6 $  {Ð3Ô4Ø�L‰LœŸ™¨¯©×)=Ñ)=Ó)?ÀÓHÔIà�y‰yÐ$Ð$Ð$Ø�‰�T×/Ñ/Ó1Ô2à�y‰y˜ÓÐrn   c                ó   — | j                   S rw   )rY  rj   s    rl   r”  zSchedulerNode.get_rangesÉ  ó   € Ø�{‰{Ðrn   c                ó<  — t        | j                  t        j                  t        j                  f«      sJ dt        | j                  «      ›�«       ‚t        | j                  j                  «       «      xr' | j                  d u xs | j                  j                   S ©Nútype(self.node)=)
rŒ   r�   r+   r   ra  r¶   rv   rÛ   rù   Úhas_partial_accumulaterj   s    rl   rŽ   zSchedulerNode.is_reductionÌ  s€   € Ü˜$Ÿ)™)¤b×&7Ñ&7¼×9JÑ9JÐ%KÔLð 	
ØŒt�D—I‘I‹Ð Ð!ó	
ÐLô �D—I‘I×0Ñ0Ó2Ó3ò 
Ø�J‰J˜$ÐÒG d§j¡j×&GÑ&GÐ"Gð	
rn   c                ó¾   — t        | j                  t        j                  «      sJ dt	        | j                  «      ›�«       ‚| j                  j                  «       dk(  S )Nr¢  Údot)rŒ   r�   r+   r   r¶   rÛ   rj   s    rl   r  zSchedulerNode.is_native_matmulÙ  sJ   € Ü˜$Ÿ)™)¤R×%6Ñ%6Ô7ÐNÐ<L¼DÀÇÁ»OÐ;MÐ9NÓNÐ7Ø�y‰y×+Ñ+Ó-°Ñ6Ð6rn   c                óL  — t        | j                  t        j                  t        j                  f«      sJ dt        | j                  «      ›�«       ‚t        | j                  t        j                  «      xr. t        | j                  j                  t        j                  «      S r¡  )rŒ   r�   r+   r   ra  r¶   rÔ   Ú	SplitScanrj   s    rl   r  zSchedulerNode.is_split_scanÝ  sy   € Ü˜$Ÿ)™)¤b×&7Ñ&7¼×9JÑ9JÐ%KÔLð 	
ØŒt�D—I‘I‹Ð Ð!ó	
ÐLô ˜$Ÿ)™)¤R×%6Ñ%6Ó7ò 
¼JØ�I‰I�N‰NœBŸL™Ló=
ð 	
rn   c                óJ   — t        | j                  t        j                  «      S rw   ©rŒ   r�   r+   ra  rj   s    rl   r   zSchedulerNode.is_templateå  s   € Ü˜$Ÿ)™)¤R×%6Ñ%6Ó7Ð7rn   c                óf   — t        | j                  t        j                  «      r| j                  S d S rw   r©  rj   s    rl   rÆ  zSchedulerNode.get_template_nodeè  s$   € Ü& t§y¡y´"×2CÑ2CÔDˆt�y‰yÐNÈ$ÐNrn   c                óf   — | j                  «        | j                  «        | j                  |«       y rw   )rL  rÑ  r;  )rk   Ú
index_varss     rl   ÚrunzSchedulerNode.runë  s#   € Ø×"Ñ"Ô$Ø�‰ŒØ�‰�ZÕ rn   c                ó&  — | j                   }t        t        t        |«      «      t        t        t        |«      «      k(  sJ ‚t	        t        t        j                  j                  |«      t        j                  j                  |«      «      «      }|S rw   )	rY  rÙ   Úmapr©   ÚdictÚziprÔ  rÕ  Úfrom_iterable)rk   r¬  r~  r´   s       rl   Úranges_from_index_varsz$SchedulerNode.ranges_from_index_varsð  sp   € ð —‘ˆÜ”3”s˜E“?Ó#¤s¬3¬s°JÓ+?Ó'@Ò@Ð@Ð@ÜÜÜ—‘×-Ñ-¨jÓ9Ü—‘×-Ñ-¨eÓ4óó
ˆ
ð Ðrn   c                ó–  — | j                  |«      }	 t        j                  t        t        j                  «       |«      «      5  t        j
                  j                  | «      5   | j                  |Ž  ddd«       ddd«       y# 1 sw Y   ŒxY w# 1 sw Y   yxY w# t        $ r" t        j                  d| j                  «       ‚ w xY w)aê  
        Generate code for this node using the provided index variables.

        This method sets up the appropriate context for code generation, including
        simplifying indexing expressions based on the variable ranges, and then
        calls the node's body function with the index variables.

        Args:
            index_vars: A sequence of sequences of sympy expressions representing
                        the index variables for each dimension of the computation.
        NzError in codegen for %s)r³  rZ   Úset_ops_handlerrG   Úget_ops_handlerr=  Úset_current_noderù   r’  r“  Úfatalr�   )rk   r¬  r´   s      rl   r;  zSchedulerNode.codegený  sª   € ð ×0Ñ0°Ó<ˆ
ð	ä×!Ñ!Ô"2´1×3DÑ3DÓ3FÈ
Ó"SÓTñ(ä—‘×)Ñ)¨$Ó/ñ(ð �—
‘
˜JÑ'÷	(÷ (÷ (ñ (ú÷ (ð (ûô
 ò 	Ü�I‰IÐ/°·±Ô;Øð	úsA   “1B Á BÁ$BÁ4BÁ<B ÂB	Â
BÂBÂB ÂB Â+Cc                óÞ   — |r| j                   nt        | j                   «      \  }}t        j                  | j                  |t
        j                  j                  gt        |«      z  g¬«      S )z\
        Get the memory dependencies in either the pointwise or the reduction axes.
        )Úhidden_args)	rY  r«   r*   rf  rù   rä   ÚSÚZeror©   )rk   Ú	pointwiseÚ
keep_sizesÚignore_sizess       rl   Ú"pointwise_or_reduction_read_writesz0SchedulerNode.pointwise_or_reduction_read_writes  sT   € ñ 3< 4§;¢;ÄÈ$Ï+É+ÓAVÑ ˆ
�LÜ×/Ñ/Ø�J‰J˜
´%·'±'·,±,°Ä#ÀlÓBSÑ1SÐ0Tô
ð 	
rn   c                ó&   — | j                  d¬«      S )zH
        Get the memory dependencies in the non-reduction axes.
        T©r½  ©rÀ  rj   s    rl   rt  z#SchedulerNode.pointwise_read_writes  s   € ð
 ×6Ñ6ÀÐ6ÓFÐFrn   c                ó&   — | j                  d¬«      S )zD
        Get the memory dependencies in the reduction axes.
        FrÂ  rÃ  rj   s    rl   Úreduction_read_writesz#SchedulerNode.reduction_read_writes&  s   € ð
 ×6Ñ6ÀÐ6ÓGÐGrn   c                óò  — | j                  «       ryt        d„ | j                  «       D «       «      ryt        | j                  j
                  «      dk(  r£t        |t        j                  «      r‰t        t        | j                  j
                  «      «      }t        |t        j                  «      sJ dt        |«      ›�«       ‚|j                  |j                  k(  xr |j                  |j                  k(  S y)NFc              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   )r(  r   s     rl   r”   z,SchedulerNode.can_inplace.<locals>.<genexpr>0  s   è ø€ Ò? Sˆs�‰× Ñ?ùrý  r   ztype(write_dep)=)r   ræ   rz  r©   r°   r�  rŒ   r*   r4   rÐ  Úiterr¶   r³   r¸   )rk   r'  Ú	write_deps      rl   r(  zSchedulerNode.can_inplace-  sÃ   € Ø×ÑÔØÜÑ?¨D×,<Ñ,<Ó,>Ô?Ô?ØÜˆt×Ñ×&Ñ&Ó'¨1Ò,´Ø”l×,Ñ,ô2
ô œT $×"2Ñ"2×"9Ñ"9Ó:Ó;ˆIÜ˜i¬×)?Ñ)?Ô@ÐWÐEUÄTÈ)Ã_ÐDVÐBWÓWÐ@Ø—>‘> Y§_¡_Ñ4ÒX¸¿¹È)Ï.É.Ñ9XÐXØrn   c                ó  — t        «       }t        | j                  t        «      rà| j                  j	                  «       D ]Ã  }|j
                  dk(  sŒ|j                  dk(  sŒ#d|j                  v r|j                  d   dk(  s,t        |j                  «      dk(  sŒ\|j                  d   dk(  sŒo|j                  d|j                  v r|j                  d   n(t        |j                  «      dk\  r|j                  d	   nd
«       ŒÅ |S )NÚcall_methodÚstoreÚmodeÚ
atomic_addr  r&  r²   r¨   r   r—  )r   rŒ   rù   r@   r–   r  rX  ru  r©   r4  rÜ  )rk   Úbuffers_store_as_atomic_addr�   s      rl   Ú_get_atomic_add_buffersz%SchedulerNode._get_atomic_add_buffers:  sÕ   € ä7A³|Ð#Ü�d—j‘j¤(Ô+ØŸ
™
×,Ñ,Ó.ò �à—G‘G˜}Ó,ØŸ™ wÓ.à 4§;¡;Ñ.°4·;±;¸vÑ3FÈ,Ò3VÜ §	¡	›N¨aÓ/°D·I±I¸a±LÀLÓ4Pð 0×3Ñ3à! T§[¡[Ñ0ð Ÿ™ FÒ+ä.1°$·)±)«nÀÒ.A˜dŸi™i¨šlÀrõðð +Ð*rn   c                óp   •— | j                   �| j                   j                  d«      ryt        ‰| �  «       S )NÚdevice_assert_asyncT)rù   Úhas_opr>  r*  ©rk   rG  s    €rl   r*  zSchedulerNode.has_side_effectsN  s2   ø€ ð �:‰:Ð! d§j¡j×&7Ñ&7Ð8MÔ&NØÜ‰wÑ'Ó)Ð)rn   )r  r  r�   z%ir.ComputedBuffer | ir.TemplateBufferr†   rU  ©NN)r^  ú'tuple[dict[Any, Any], list[Any]] | Noner_  zCallable[_P, _T] | Noner†   rU  )r^  rÖ  r_  zCallable[..., Any] | Noner†   rU  )r`  rv   ry  rv   r†   rU  )r  úSequence[int]r†   rU  rT  ©r†   r_   )r‘  r  r’  r  r†   rU  rÙ  rR  rP  rV  rå  )r¬  úSequence[sympy.Expr]r†   rU  )r¬  rQ  r†   zdict[sympy.Expr, sympy.Expr])r¬  rQ  r†   rU  rá  )r½  rv   r†   ro  )r†   ro  rà  rÜ  )"ry   rz   r{   r  r|   rx  r[  ro  rz  r€  r‰  r‹  rŽ  r�  r”  r¯  r‘  r”  rŽ   r  r  r   rÆ  r­  r³  r;  rÀ  rI   rt  rÅ  r(  rÐ  r*  rS  rT  s   @rl   r�   r�      sº  ø… ñð
 -Ó,ØƒOðàðð 4ðð 
õ	ð OSØ=Aðà$Kðð $;ðð 
ó	ðF OSØ?Cðà$Kðð $=ðð 
ó	ð";Øð;Ø8<ð;à	ó;ó<QóIóó"ðPØðPØ),ðPà	óPó"
QðØ!ðØ.7ðà	óó. ó,ó
ó7ó
ó8óOó!ð
Ø8ðà	%óóð0 !%ð	
Øð	
à	 ó	
ð òGó ðGð òHó ðHóð ò+ó ð+ð& ô*ó ô*rn   r�   c           	     ón  ‡ — ‰ j                   }‰ j                  t        j                  j	                  |D �cg c]  }|j
                  ‘Œ c}«      «       t        ˆ fd„t        j                  |D �cg c]  }|j                  ‘Œ c}Ž D «       «      ‰ j
                  j                  z
  ‰ _        y c c}w c c}w )Nc              3  óZ   •K  — | ]"  }|j                   ‰j                  «       vr|–— Œ$ y ­wrw   ©r²   r  )r’   r¼   Úgroup_snodes     €rl   r”   z2refresh_group_node_dependencies.<locals>.<genexpr>_  s.   øè ø€ ò 
àØ�x‰x˜{×;Ñ;Ó=Ñ=ô ñ
ùó   ƒ(+)
r·   r·  r*   Ú
ReadWritesÚ
merge_listr°   r   Úunionrq  r�  )rÝ  r·   rJ  s   `  rl   Úrefresh_group_node_dependenciesrâ  V  sž   ø€ ð ×Ñ€FØ×ÑÜ×Ñ×*Ñ*À6Ö+J¸a¨A¯M«MÒ+JÓKôô
 	ó 
ä!×'Ñ'ÈÖ)OÀ1¨!×*>Ó*>Ò)OÐPô
ó 	
ð
 ×
!Ñ
!×
(Ñ
(ñ	)ð Õ"ùò ,Kùò *Ps   ¶B-Á0B2r  c                ól  — t        | t        t        f«      sJ ‚|| _        || _        d | _        t        j                  |D �cg c]  }|j                  €Œ|j                  ‘Œ c}Ž | _        t        | «       t        d„ | j                  D «       «      | _        t        d„ | j                  D «       «      | _        t        d„ | j                  D «       «      | _        t        d„ | j                  D «       «      | _        | j!                  «       D �ci c]  }|j#                  «       |“Œ c}| _        y c c}w c c}w )Nc              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   ©rc  ©r’   rJ  s     rl   r”   z"init_group_node.<locals>.<genexpr>w  ó   è ø€ ÒH° §¥ÑHùrµ  c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   )rd  ræ  s     rl   r”   z"init_group_node.<locals>.<genexpr>x  rç  rµ  c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   )ra  ræ  s     rl   r”   z"init_group_node.<locals>.<genexpr>y  ó   è ø€ ò )Ø!"ˆ×Õñ)ùrµ  c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   )rb  ræ  s     rl   r”   z"init_group_node.<locals>.<genexpr>|  rê  rµ  )rŒ   rµ   ÚGroupedSchedulerNoder·   r  r�   r   rá  rß   râ  r‰  rc  r¶  rd  ra  rb  rz  r  rl  )rÝ  r  r·   rJ  rº   s        rl   Úinit_group_noderí  h  s  € ô
 �kÔ$6Ô8LÐ#MÔNÐNÐNØ€KÔØ%€KÔØ€KÔÜ&×,Ñ,Ø%Ö	A˜!¨¯©Ñ)@ˆ!�+‹+Ò	Að€KÔô $ KÔ0äÑH°[×5GÑ5GÔHÓH€KÔÜÑH°[×5GÑ5GÔHÓH€KÔÜ%(ñ )Ø&1×&8Ñ&8ô)ó &€KÔ"ô &)ñ )Ø&1×&8Ñ&8ô)ó &€KÔ"ð (3×'>Ñ'>Ó'@ö#Ø #ˆ�‰‹˜Ñò#€KÕùò 
Bùò#s   ÁD,ÁD,ÄD1c                  ó¾  ‡ — e Zd ZU dZded<   e	 	 	 	 	 	 d!d„«       Zd"d„Zd#d„Ze	d$d„«       Z
	 	 	 	 	 	 d%d„Zd&ˆ fd	„Ze	d'd
„«       Zd'd„Ze	d(d„«       Zd)d„Zd'd„Zd'd„Z	 	 	 	 	 	 d*ˆ fd„Ze	d(d„«       Ze	d(d„«       Zd+d„Zd'd„Ze	d,d„«       Ze	d,d„«       Ze	d,d„«       Ze	d,d„«       Ze	d-d„«       Zd.d„Ze	d,d„«       Zd/d„Zd0d„Z d1d„Z!d'd„Z"e	d,ˆ fd „«       Z#ˆ xZ$S )2rµ   zæ
    This is a "fake" scheduler node that represents a group of scheduler nodes
    that are meant to be fused together. The way it does this is by maintaining
    its unmet dependencies as the union of its constituent nodes.
    ræ  r·   c                ó¾  — |j                   |j                   u sJ ‚t        |t        t        f«      sJ ‚|j	                  «       r6t        |t
        «      r&t        |j                  t        j                  «      sJ ‚t        |t        t        f«      sJ ‚t        t        j                  |j                  «       |j                  «       «      «      } | |j                   |«      S rw   )r  rŒ   r�   rµ   r   r�  r�   r+   r=   rÿ   rÔ  rÕ  r–   )rq   r�   r‚   rÒ  s       rl   rr   zFusedSchedulerNode.fuse�  s¨   € ð �‰ %§/¡/Ñ1Ð1Ð1Ü˜%¤-Ô1CÐ!DÔEÐEÐEØ×ÑÔ¤:¨eÔ5NÔ#OÜ˜eŸj™j¬"¯.©.Ô9Ð9Ð9ä˜e¤mÔ5GÐ%HÔIÐIÐIÜ”Y—_‘_ U§_¡_Ó%6¸¿¹Ó8IÓJÓKˆÙ�5—?‘? EÓ*Ð*rn   c                ó�   — | j                   D ]6  }t        |t        «      sJ ‚|j                  «       sJ ‚|j	                  «        Œ8 | S rw   )r·   rŒ   r�   rŽ   r‹  ©rk   r“   s     rl   r‹  z,FusedSchedulerNode.extract_pw_from_reductionš  sJ   € Ø—{‘{ò 	0ˆGÜ˜g¤}Ô5Ð5Ð5Ø×'Ñ'Ô)Ð)Ð)Ø×-Ñ-Õ/ð	0ð ˆrn   c                ój   — | j                   D ]$  }t        |t        «      sJ ‚|j                  «        Œ& y rw   )r·   rŒ   r�   r‰  rñ  s     rl   r‰  z(FusedSchedulerNode.swap_pw_red_dimension¡  s1   € Ø—{‘{ò 	,ˆGÜ˜g¤}Ô5Ð5Ð5Ø×)Ñ)Õ+ñ	,rn   c                óŽ   — t        t        d d„ | j                  «       D «       «      «      }t        |«      dk(  ry t	        |«      }|S )Nc              3  ó|   K  — | ]4  }|j                  «       s|j                  «       r|j                  «       –— Œ6 y ­wrw   ©r   r"  r£  rü  s     rl   r”   z4FusedSchedulerNode.estimate_flops.<locals>.<genexpr>¬  ó6   è ø€ ò àØ×'Ñ'Ô)¨T¯^©^Ô-=ð ×'Ñ'×)ñùó   ‚:<r   ©rÿ   Úfilterr–   r©   rÙ   ©rk   Úfpsr»  s      rl   r£  z!FusedSchedulerNode.estimate_flops¦  óK   € ô ÜØñà $§¡Ó 0ôóó	
ˆô ˆs‹8�qŠ=ØÜ�#‹hˆØˆ
rn   c                óÔ  — | j                  «       ryd}| j                  D ]`  }t        |t        «      s y|�;t	        |«      t	        |j
                  d   «      k7  rt        j                  d«        y|j
                  d   }Œb d}|€J ‚t        |«      |j                  cxk(  r|j                  k(  rn n|j                  |«      }|s%t        j                  d| j                  «       «       yt        xj                  dz  c_        t        j                  d| j                  «       |«       | j                  D ]%  }t        |t        «      sJ ‚|j                  |«       Œ' t        | «       y)	z@
        Return true if a loop reordering is performed.
        FNr   z1Can not reorder fused node due to different sizeszODont reordering fused node %s because we can not decide the suitable loop orderr   z-Reorder loops for fused node %s with order %sT)r   r·   rŒ   r�   rª   rY  r™  r  r©   r–  r—  r  r,   r˜  r€  râ  )rk   r­  r®  rš  rø  r  s         rl   r¯  z,FusedSchedulerNode.reorder_loops_by_dep_pair¸  sI  € ð ×ÑÔàØˆ
Ø—[‘[ò 	)ˆEÜ˜e¤]Ô3ÙØÐ%¬%°
Ó*;¼uÀUÇ\Á\ÐRSÁ_Ó?UÒ*UÜ!×'Ñ'ØGôñ ØŸ™ a™‰Jð	)ð ˆ	àÐ%Ð%Ð%Üˆz‹?˜h×/Ñ/ÔE°9×3EÑ3EÕEØ ×;Ñ;¸IÓFˆIáÜ×#Ñ#ØaØ—‘“ôð ä×#Ò# qÑ(Õ#Ü×ÑØ;¸T¿]¹]»_Èiô	
ð —[‘[ò 	2ˆEÜ˜e¤]Ô3Ð3Ð3Ø×&Ñ& yÕ1ð	2ô 	(¨Ô-Ørn   c                ó†   •— t         ‰| �  |«       t        | ||«       g | _        t	        |d„ ¬«      j
                  | _        y )Nc                ó4   — t        | j                  «       «      S rw   )r  rŽ   rï  s    rl   rv  z-FusedSchedulerNode.__init__.<locals>.<lambda>ç  s   € ¬s°1·>±>Ó3CÓ/D€ rn   r$  )r>  rx  rí  r  r¶  r¡   )rk   r  r·   rG  s      €rl   rx  zFusedSchedulerNode.__init__ã  s8   ø€ Ü‰Ñ˜Ô#Ü˜˜i¨Ô0Ø%'ˆŒ
Ü˜Ñ%DÔE×KÑKˆ�
rn   c                óz   — dj                  | j                  D �cg c]  }|j                  «       ‘Œ c}«      S c c}w ©NÚ_©rœ  r·   r  ©rk   rJ  s     rl   r  zFusedSchedulerNode.get_nameé  ó)   € à�x‰x¨t¯{©{Ö;¨!˜Ÿ™�Ò;Ó<Ð<ùÒ;ó   š8c                ó<   — | j                   d   j                  «       S rÅ   ©r·   r  rj   s    rl   rù  z!FusedSchedulerNode.get_first_nameí  ó   € Ø�{‰{˜1‰~×&Ñ&Ó(Ð(rn   c                ó|   — t        j                  | j                  D �cg c]  }|j                  «       ‘Œ c}Ž S c c}w rw   ©r   rá  r·   r  r  s     rl   r  z#FusedSchedulerNode.get_buffer_namesð  ó.   € ä×ÑÀÇÁÖ!L¸1 !×"4Ñ"4Õ"6Ò!LÐMÐMùÒ!Ló   ž9c                ój   — g }| j                   D ]!  }|j                  |j                  «       «       Œ# |S rw   ©r·   rÞ  rz  ©rk   r-  r�   s      rl   rz  zFusedSchedulerNode.get_outputsô  ó4   € Ø(*ˆØ—K‘Kò 	.ˆDØ�M‰M˜$×*Ñ*Ó,Õ-ð	.àˆrn   c           
     ó~  — t        | j                  «      D ��cg c]+  \  }}| j                  «       › d|› d|j                  «       › �‘Œ- }}}| j                  d   j                  }|�|j                  | j                  «       «       t        j                  dj                  |«      j                  «       d«      S c c}}w )Nz.snodes[z] =
r   r†  r'  )rÑ  r·   r  r/  r�   rÞ  rš  r+  r+  rœ  r•  )rk   rÏ  r�   r�  s       rl   r‘  z"FusedSchedulerNode.debug_str_extraú  s¦   € ô % T§[¡[Ó1÷
á��4ð �}‰}‹Ð˜x¨ s¨%°·±Ó0@Ð/AÒBð
ˆñ 
ð �{‰{˜1‰~×"Ñ"ˆØÐØ�L‰L˜×3Ñ3Ó5Ô6ä�‰˜tŸy™y¨Ó/×6Ñ6Ó8¸&ÓAÐAùó
s   ™0B9c                óh   — | j                   D �cg c]  }|j                  «       ‘Œ }}| › d|› �S c c}w )Nz
, snodes: )r·   r§  )rk   r�   Ú
snodes_strs      rl   r§  z"FusedSchedulerNode.debug_str_short  s9   € Ø9=¿¹ÖE°�d×*Ñ*Õ,ÐEˆ
ÐEØ��z * Ð.Ð.ùò Fs   �/c                óÈ   •— t         ‰| �  ||«       t        «       }t        | j                  «      D ]/  }|j                  ||«       |j                  |j                  «       Œ1 y rw   )r>  rÏ  r   r«   r·   Úupdater`  )rk   rÍ  rÊ  r�   rG  s       €rl   rÏ  z!FusedSchedulerNode.set_last_usage	  s\   ø€ ô
 	‰ÑÐ2Ð4FÔGô 0:«|ÐÜ˜TŸ[™[Ó)ò 	8ˆDØ×ÑÐ 3Ð5GÔHØ×&Ñ& t§¡Õ7ñ	8rn   c                ó|   — t        j                  | j                  D �cg c]  }|j                  «       ‘Œ c}Ž S c c}w rw   )r   rá  r·   r¿   r  s     rl   r¿   z$FusedSchedulerNode.used_buffer_names  s.   € ä×ÑÀÇÁÖ!M¸A !×"5Ñ"5Õ"7Ò!MÐNÐNùÒ!Mr  c                ó|   — t        j                  | j                  D �cg c]  }|j                  «       ‘Œ c}Ž S c c}w rw   )r   rá  r·   rÌ  r  s     rl   rÌ  z/FusedSchedulerNode.used_or_aliased_buffer_names  s3   € ä×ÑØ8<¿¹ÖD°1ˆa×,Ñ,Õ.ÒDð
ð 	
ùÚDr  c                ó   — | j                   S rw   r  rj   s    rl   r–   zFusedSchedulerNode.get_nodes   rŸ  rn   c                óT   — t        | «      j                  › d| j                  «       › d�S )Nz(nodes=r~  r  rj   s    rl   r€  zFusedSchedulerNode.__repr__#  s'   € Ü�t“*×%Ñ%Ð& g¨d¯m©m«oÐ->¸aÐ@Ð@rn   c                ó:   — t        d„ | j                  D «       «      S )Nc              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   )rŽ   ræ  s     rl   r”   z2FusedSchedulerNode.is_reduction.<locals>.<genexpr>(  s   è ø€ Ò9¨�1—>‘>×#Ñ9ùrý  ©ræ   r·   rj   s    rl   rŽ   zFusedSchedulerNode.is_reduction&  s   € äÑ9¨T¯[©[Ô9Ó9Ð9rn   c                ó:   — t        d„ | j                  D «       «      S )Nc              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   )r  ræ  s     rl   r”   z6FusedSchedulerNode.is_native_matmul.<locals>.<genexpr>,  s   è ø€ Ò=¨A�1×%Ñ%×'Ñ=ùrý  r  rj   s    rl   r  z#FusedSchedulerNode.is_native_matmul*  s   € äÑ=°·±Ô=Ó=Ð=rn   c                ó:   — t        d„ | j                  D «       «      S )Nc              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   )r  ræ  s     rl   r”   z3FusedSchedulerNode.is_split_scan.<locals>.<genexpr>0  s   è ø€ Ò:¨�1—?‘?×$Ñ:ùrý  r  rj   s    rl   r  z FusedSchedulerNode.is_split_scan.  s   € äÑ:¨d¯k©kÔ:Ó:Ð:rn   c                ó:   — t        d„ | j                  D «       «      S )Nc              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   rÎ  ræ  s     rl   r”   z1FusedSchedulerNode.is_template.<locals>.<genexpr>4  s   è ø€ Ò8 q�1—=‘=—?Ñ8ùrý  r  rj   s    rl   r   zFusedSchedulerNode.is_template2  s   € äÑ8¨D¯K©KÔ8Ó8Ð8rn   c                ój   — | j                   D ]$  }|j                  «       sŒ|j                  «       c S  y rw   )r·   r   rÆ  ©rk   r�   s     rl   rÆ  z$FusedSchedulerNode.get_template_node6  s5   € à—K‘Kò 	0ˆDØ×ÑÕ!Ø×-Ñ-Ó/Ò/ð	0ð rn   c                ó    — | j                   d   S rÅ   )r¡   rj   s    rl   rÞ   zFusedSchedulerNode.get_device=  s   € Ø�z‰z˜!‰}Ðrn   c                ó:   — t        d„ | j                  D «       «      S )Nc              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   )rÂ  ræ  s     rl   r”   z>FusedSchedulerNode.has_aliasing_or_mutation.<locals>.<genexpr>B  s   è ø€ ÒE°A�1×-Ñ-×/ÑEùrý  r  rj   s    rl   rÂ  z+FusedSchedulerNode.has_aliasing_or_mutation@  s   € äÑE¸¿¹ÔEÓEÐErn   c                ó   — t         ‚rw   ©ÚNotImplementedError)rk   r¹  s     rl   rº  z'FusedSchedulerNode.update_mutated_namesF  ó   € Ü!Ð!rn   c                ó   — t         ‚rw   r*  )rk   r²   s     rl   r¾  zFusedSchedulerNode.add_fake_depI  r,  rn   c                ó   — t         ‚rw   r*  r&  s     rl   r(  zFusedSchedulerNode.can_inplaceL  r,  rn   c                ó�  — | j                  «       }dj                  d„ | j                  D «       «      }t        «       }|j	                  |› dt        | «      j                  › d|› d|› dt        | j                  j                  «      › d|› dt        | j                  «      › d|› d	t        | j                  j                  | j                  z
  «      › d|› d
| j                  › d|› d| j                  › d|› d�«       |j                  «       5  | j                  «       D ]!  }|j	                  |j!                  «       «       Œ# 	 ddd«       |j#                  d«       	 |j	                  | j%                  «       «       |j-                  «       j/                  «       S # 1 sw Y   ŒXxY w# t&        $ r t(        j+                  dd¬«       Y ŒLw xY w)r‚  r$  c              3  óF   K  — | ]  }t        |«      j                  –— Œ y ­wrw   )r¶   ry   r  s     rl   r”   z/FusedSchedulerNode.debug_str.<locals>.<genexpr>R  s   è ø€ ÒF°Q¤ Q£× 0Õ 0ÑFùs   ‚!r#  rƒ  r„  r…  r†  r‡  rˆ  r‰  rŠ  z.outputs = [
            Nr%  r‹  TrŒ  )r  rœ  r·   rR   rŽ  r¶   ry   r)  r°   r�  rq  r±   ra  rb  r+  rz  r/  r&  r‘  r’  r“  r”  r,  r•  )rk   r²   Únode_typestrrº   rÁ   s        rl   r/  zFusedSchedulerNode.debug_strO  sµ  € à�}‰}‹ˆØ—x‘xÑF¸$¿+¹+ÔFÓFˆÜÓˆØ�
‰
à€€bŒˆd‹×	Ñ	Ð˜Q˜|˜nð -Ø€€j”˜×)Ñ)×0Ñ0Ó1Ð2ð 3Ø€ÐœW T×%<Ñ%<Ó=Ð>ð ?Ø€Ðœ7 4×#3Ñ#3×#9Ñ#9¸D×<SÑ<SÑ#SÓTÐUð VØ€Ð˜T×4Ñ4Ð5ð 6Ø€Ð˜T×4Ñ4Ð5ð 6Ø€ð ðô
	
ð �Z‰Z‹\ñ 	,Ø×'Ñ'Ó)ò ,�Ø—
‘
˜3Ÿ=™=›?Õ+ñ,÷	,ð 	�‰�cÔð	HØ�J‰J�t×+Ñ+Ó-Ô.ð �‰Ó ×'Ñ'Ó)Ð)÷	,ð 	,ûô ò 	HÜ�K‰KÐ7À$ˆKÖGð	Hús   Ä	5FÅF" ÆFÆ" GÇGc                óp   •— | j                   �t        d„ | j                   D «       «      S t        ‰| �  «       S )Nc              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   )r*  rü  s     rl   r”   z6FusedSchedulerNode.has_side_effects.<locals>.<genexpr>n  s   è ø€ ÒG°4�t×,Ñ,×.ÑGùrý  )r·   ræ   r>  r*  rÔ  s    €rl   r*  z#FusedSchedulerNode.has_side_effectsk  s0   ø€ à�;‰;Ð"ÜÑG¸4¿;¹;ÔGÓGÐGÜ‰wÑ'Ó)Ð)rn   ©r�   r_   r‚   r_   r†   rµ   rØ  rT  râ  rÙ  )r  r  r·   ræ  r†   rU  rR  rÜ  ©r†   ri  rÛ  rß  rV  rå  )r†   útorch.devicerÚ  )r²   r3   r†   rU  rà  )%ry   rz   r{   r  r|   r}   rr   r‹  r‰  rI   r£  r¯  rx  r  rù  r  rz  r‘  r§  rÏ  r¿   rÌ  r–   r€  rŽ   r  r  r   rÆ  rÞ   rÂ  rº  r¾  r(  r/  r*  rS  rT  s   @rl   rµ   rµ   „  s´  ø… ñð $Ó#àð
+Ø%ð
+Ø.?ð
+à	ò
+ó ð
+óó,ð
 òó ðð")Ø!ð)Ø.7ð)à	ó)õVLð ò=ó ð=ó)ð òNó ðNóó	Bó/ð8Ø#2ð8ØHVð8à	õ8ð òOó ðOð ò
ó ð
ó
óAð ò:ó ð:ð ò>ó ð>ð ò;ó ð;ð ò9ó ð9ð òó ðóð òFó ðFó
"ó"ó"ó*ð8 ô*ó ô*rn   rµ   c                  óD   ‡ — e Zd Zdˆ fd„Z	 	 	 	 	 	 dd„Zdd„Zdd„Zˆ xZS )ÚFusedMixOrderReductionsc                ó`  •— t         j                  |«      st         j                  |«      sJ ‚||}}|| _        || _        t        ‰| �  |j                  t        |j                  «       «      t        |j                  «       «      z   «       t         j                  | j                  «      | _
        y rw   )r‰   rá   r�   r‚   r>  rx  r  rÿ   r–   rÌ   rF  )rk   r�   r‚   rG  s      €rl   rx  z FusedMixOrderReductions.__init__s  s‡   ø€ Ü ×3Ñ3°EÔ:Ü$×7Ñ7¸Ô>Ð>Ð>Ø  %�5ˆEàˆŒ
ØˆŒ
Ü‰ÑØ�O‰OœT %§/¡/Ó"3Ó4´t¸E¿O¹OÓ<MÓ7NÑNô	
ô '×0Ñ0°·±Ó<ˆ�
rn   c                óô  — t        |t        «      rJ ‚t        |t        «      rJ ‚| j                  j                  ||d¬«      syt        j                  |«      rt        j                  |«      sydd„}	 	 	 	 dd„}|r' |||f«       ||«      z  s ||«       |||f«      z  ry|j                  «        xsC t        j                  t        | j                  j                  ||d¬«      «      | j                  k\  S )a  
        node1 is from the current mix order reduction; node2 is another node we want to fuse in.

        other_nodes are passed in to check if fusion will introduce producer/consumer relationship
        between the inner and outer reduction. If yes, we don't fuse.
        F©Úallow_mix_order_reductionc                óB   — t        «       } |j                  d„ | D «       Ž S )Nc              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   )rß   r  s     rl   r”   zTFusedMixOrderReductions.sub_node_can_fuse.<locals>._get_ancestors.<locals>.<genexpr>�  s   è ø€ Ò:¨q˜qŸ{�{Ñ:ùrµ  ©r   rá  ©rÒ  rÁ   s     rl   Ú_get_ancestorszAFusedMixOrderReductions.sub_node_can_fuse.<locals>._get_ancestors›  s    € Ü“,ˆCØ�3—9‘9Ñ:°EÔ:Ð;Ð;rn   c                óB   — t        «       } |j                  d„ | D «       Ž S )Nc              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   )rà   r  s     rl   r”   zZFusedMixOrderReductions.sub_node_can_fuse.<locals>._get_operation_names.<locals>.<genexpr>£  s   è ø€ ÒF¸1˜q×4Ñ4×6ÑFùrý  r?  r@  s     rl   Ú_get_operation_nameszGFusedMixOrderReductions.sub_node_can_fuse.<locals>._get_operation_namesŸ  s"   € ô “,ˆCØ�3—9‘9ÑFÀÔFÐGÐGrn   )Úcount_bytes)rÒ  útuple[BaseSchedulerNode, ...]r†   r_  )rŒ   r8  r  rî   r‰   rá   rŽ   ÚtypingÚcastr  Úscore_fusion_memoryrF  )rk   r�   r‚   Úother_nodesrA  rD  s         rl   Úsub_node_can_fusez)FusedMixOrderReductions.sub_node_can_fuse  s  € ô ˜eÔ%<Ô=Ð=Ð=Ü˜eÔ%<Ô=Ð=Ð=ð
 �~‰~×&Ñ& u¨eÈuÐ&ÔUØô ×/Ñ/Øô
ä#×6Ñ6°uÔ=Øó	<ð	HØ0ð	Hàó	Hñ Ù  u˜~Ó.Ñ1EÀkÓ1RÒRÙ˜{Ó+Ñ.BÀEÈ5À>Ó.RÒRàð ×"Ñ"Ó$Ð$ò Ü�{‰{Ü�T—^‘^×7Ñ7¸¸uÐRWÐ7ÓXóð �z‰zñð	
rn   c                ó¢  — t        |t        «      sR| j                  | j                  || j                  f«      xs( | j                  | j                  || j                  f«      S | j                  | j                  |j                  | j                  |j                  f«      xr/ | j                  | j                  |j                  t        «       «      S rw   )rŒ   r8  rK  r�   r‚   rª   ©rk   Úothers     rl   Úcan_fuse_withz%FusedMixOrderReductions.can_fuse_with³  s¨   € Ü˜%Ô!8Ô9Ø×)Ñ)Ø—
‘
˜E D§J¡J =óò Jà×'Ñ'¨¯
©
°E¸D¿J¹J¸=ÓIðJð ×)Ñ)Ø—
‘
˜EŸK™K¨$¯*©*°e·k±kÐ)Bóò Kà×(Ñ(¨¯©°U·[±[Ä%Ã'ÓJðKrn   c                óT  — | j                   j                  «       }| j                  j                  |«      }t	        |t
        «      rX|j                  | j                   |j                   «      }|j                  | j                  |j                  «      }t        ||«      S | j                  | j                   || j                  f«      r2|j                  | j                   |«      }t        || j                  «      S |j                  | j                  |«      }t        | j                   |«      S rw   )	r�   rÞ   r  rc  rŒ   r8  rr   r‚   rK  )rk   rN  r  ÚbackendÚfused_node1Úfused_node2r4  s          rl   Ú	fuse_withz!FusedMixOrderReductions.fuse_with¿  sÖ   € Ø—‘×&Ñ&Ó(ˆØ—.‘.×,Ñ,¨VÓ4ˆä�eÔ4Ô5Ø!Ÿ,™, t§z¡z°5·;±;Ó?ˆKØ!Ÿ,™, t§z¡z°5·;±;Ó?ˆKÜ*¨;¸ÓDÐDà×%Ñ% d§j¡j°%¸$¿*¹*¸ÔGØ$Ÿ\™\¨$¯*©*°eÓ<�
Ü.¨z¸4¿:¹:ÓFÐFà$Ÿ\™\¨$¯*©*°eÓ<�
Ü.¨t¯z©z¸:ÓFÐFrn   r"  )r�   r_   r‚   r_   rJ  rF  )rN  r_   )ry   rz   r{   rx  rK  rO  rT  rS  rT  s   @rl   r8  r8  r  s6   ø„ õ
=ð2
à ð2
ð !ð2
ð 3ó	2
óh
K÷Grn   r8  c                  óf   ‡ — e Zd Z	 	 	 	 	 	 	 	 dˆ fd„Ze	 	 	 	 	 	 dd„«       Zdd„Zd	d„Zd
d„Zˆ xZ	S )Ú$FusedExternTritonKernelSchedulerNodec                ó,  •— t        |j                  t        j                  «      sJ ‚t	        j
                  t        t           ||g«      }t        ‰| �%  ||«       || _
        || _        | j                  j                  | _        |j                  | _        y rw   )rŒ   r�   r+   r@  rG  rH  rÿ   r_   r>  rx  Úkernel_nodeÚfused_epiloguerc  rj  )rk   r  rX  rY  r·   rG  s        €rl   rx  z-FusedExternTritonKernelSchedulerNode.__init__Ñ  s{   ø€ ô ˜+×*Ñ*¬B×,FÑ,FÔGÐGÐGÜ—‘œTÔ"3Ñ4°{ÀNÐ6SÓTˆÜ‰Ñ˜ FÔ+Ø&ˆÔØ,ˆÔØ×)Ñ)×3Ñ3ˆŒØ%×-Ñ-ˆ�rn   c                ó  — |j                   }t        |j                  «      dk(  sJ ‚|j                  t	        t        |j                  «      «      j                     }|j                  j                  t        |«      «       t        |||«      S ©Nr   )r  r©   rq  r@  rÐ  rÈ  r²   r  ÚremoveÚNodeUserrV  )rq   r�   r‚   r  Úoriginal_mutated_buffers        rl   Úepilogue_fusez2FusedExternTritonKernelSchedulerNode.epilogue_fuseß  s{   € ð —O‘Oˆ	ô �5×+Ñ+Ó,°Ò1Ð1Ð1Ø"+×"7Ñ"7Ü”�e×.Ñ.Ó/Ó0×5Ñ5ñ#
Ðð 	 ×%Ñ%×,Ñ,¬X°e«_Ô=Ü3°I¸uÀeÓLÐLrn   c                óÆ  — t        | j                  j                  t        j                  «      sJ ‚t        | j
                  j                  t        j                  «      sJ ‚| j
                  j                  j                  «       sJ ‚t        j                  | j
                  j                  j                  d   j                  «      }ddlm} |j                  | j                  g|«      \  }}ddlm}  || j                  g|«      }ddlm}  |||| «      }	|	j'                  «       }
| j
                  j                  j)                  || j                  j                  |
f«      S )Nr   rr  )ÚSIMDKernelFeatures)ÚFusedUserDefinedTritonKernel)rŒ   rY  r�   r+   r   rX  r@  rA  rB  rÚ   rC  rD  Útorch._inductor.codegen.simdrs  Úget_tiling_and_scoresÚ,torch._inductor.codegen.simd_kernel_featuresra  Útorch._inductor.codegen.tritonrb  r;  Úcodegen_with_epilogue_fusion)rk   rN  rF  rs  Útilingr  ra  Úkernel_featuresrb  Úfused_user_kernelÚnew_kernel_srcs              rl   r;  z,FusedExternTritonKernelSchedulerNode.codegenð  s   € Ü˜$×-Ñ-×2Ñ2´B×4EÑ4EÔFÐFÐFÜ˜$×*Ñ*×/Ñ/´×1KÑ1KÔLÐLÐLØ×Ñ×$Ñ$×6Ñ6Ô8Ð8Ð8Ü—	‘	˜$×*Ñ*×/Ñ/×<Ñ<¸QÑ?×EÑEÓFˆÝ?à"×8Ñ8¸$×:MÑ:MÐ9NÐPUÓV‰	ˆ�åSá,¨d×.AÑ.AÐ-BÀEÓJˆåOá8¸ÀÐRVÓWÐØ*×2Ñ2Ó4ˆà×Ñ×$Ñ$×AÑAØ�d×)Ñ)×.Ñ.°Ð?ó
ð 	
rn   c                 ó   — yrJ  r~   rj   s    rl   r"  z.FusedExternTritonKernelSchedulerNode.is_extern	  rÇ  rn   c                ó6   — | j                   j                  «       S rw   )rX  r”  rj   s    rl   r”  z/FusedExternTritonKernelSchedulerNode.get_ranges		  s   € Ø×Ñ×*Ñ*Ó,Ð,rn   )r  r  rX  r�  rY  r�   r†   rU  )r�   r�  r‚   r�   r†   rµ   rR  rV  rP  )
ry   rz   r{   rx  r}   r_  r;  r"  r”  rS  rT  s   @rl   rV  rV  Ð  sr   ø„ ð.àð.ð /ð.ð &ð	.ð
 
õ.ð ðMà(ðMð ðMð 
ò	Mó ðMó 
ó,÷-rn   rV  c                  óN  ‡ — e Zd ZU dZ	 	 	 	 dd„Z	 	 	 	 dd„Zedd„«       Ze	 	 	 	 	 	 dd„«       Z	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Z	e	 	 	 	 dd„«       Z
e	 	 	 	 dd„«       ZeZd	ed
<   e	 	 	 	 dd„«       Ze	 	 	 	 dd„«       Zdd„Zdd„Zdd„Zdd„Zd d„Zd!d„Z	 	 	 	 d"d„Zˆ xZS )#ÚForeachKernelSchedulerNodezš
    This is a schedular node that consists of a set of scheduler nodes that
    has no data dependencies among them and can be executed in parallel.
    c                ó¤   — |j                  «       D ]=  }|j                  «       | j                  v sŒ | j                  |j                  «          c S  y rw   )rz  r  Úread_to_node)rk   Úproducerrº   s      rl   Úget_consumer_subnode_forz3ForeachKernelSchedulerNode.get_consumer_subnode_for	  sL   € ð ×'Ñ'Ó)ò 	9ˆCØ�|‰|‹~ ×!2Ñ!2Ò2Ø×(Ñ(¨¯©«Ñ8Ò8ð	9ð rn   c                ó¢  — t        t           «       }|j                  j                  D ]ƒ  }|j                  | j
                  j                  vrŒ&| j
                  j                  |j                     j                  «       }|| j                  v sŒf|j                  | j                  |   «       Œ… t        |«      dk(  rt        t        |«      «      S y r[  )r   r_   r°   r±   r²   r  r@  r  Úname_to_noderÜ  r©   rÐ  rÈ  )rk   ÚconsumerÚ	producersÚrdÚ	node_names        rl   Úget_producer_subnode_forz3ForeachKernelSchedulerNode.get_producer_subnode_for	  s¬   € ô Ô0Ñ1Ó3ˆ	Ø×&Ñ&×,Ñ,ò 	<ˆBØ�w‰w˜dŸn™n×8Ñ8Ñ8ØàŸ™×2Ñ2°2·7±7Ñ;×LÑLÓNˆIØ˜D×-Ñ-Ò-Ø—‘˜d×/Ñ/°	Ñ:Õ;ð	<ô ˆy‹>˜QÒÜœ˜Y›Ó(Ð(àrn   c                óœ  ‡— t        ‰|«      }‰j                  «       r¯|j                  «       rŸt        j                  t        ‰«      Št        j                  t        |«      }t        ‰j                  «      t        |j                  «      k(  }|s |d«       |xr2 t        ˆfd„t        ‰j                  |j                  «      D «       «      S |j                  «       rk‰j                  «       r	 |d«       yt        j                  t        |«      }|j                  ‰«      }|�|j                  j                  ‰|«      S  |d«       y‰j                  «       rk|j                  «       r	 |d«       yt        j                  t        ‰«      Š‰j                  |«      }|�‰j                  j                  ||«      S  |d«       yt        d«      ‚)	Nzforeach do not have same lengthc              3  ó\   •K  — | ]#  \  }}‰j                   j                  ||«      –— Œ% y ­wrw   )r  rî   )r’   ÚlÚrrr  s      €rl   r”   z6ForeachKernelSchedulerNode.can_fuse.<locals>.<genexpr>8	  s0   øè ø€ ò )á�A�qð ×"Ñ"×+Ñ+¨A¨q×1ñ)ùs   ƒ),zXcandidate producer is a reduction, foreach ops cannot be fused with reductions currentlyFz5candidate producer is not dep of any foreach consumerzXcandidate consumer is a reduction, foreach ops cannot be fused with reductions currentlyz5candidate consumer has no dep in any foreach producerzXAt least one node passed to ForeachKernelSchedulerNode.can_fuse should be a foreach node)r  r$  rG  rH  ro  r©   r·   r•   r±  rŽ   rs  r  rî   rz  rµ  )rq   rr  rv  ÚwhyÚforeach_matchÚconsumer_subnodeÚproducer_subnodes    `     rl   rî   z#ForeachKernelSchedulerNode.can_fuse/	  sž  ø€ ä˜ (Ó+ˆØ×ÑÔ  X×%8Ñ%8Ô%:Ü—{‘{Ô#=¸xÓHˆHÜ—{‘{Ô#=¸xÓHˆHÜ §¡Ó0´C¸¿¹Ó4HÑHˆMÙ ÙÐ5Ô6Ø ò ¤Só )ä §¡°·±ÓAô)ó &ð ð × Ñ Ô"Ø×$Ñ$Ô&ÙØnôð ä—{‘{Ô#=¸xÓHˆHØ'×@Ñ@ÀÓJÐØÐ+Ø×)Ñ)×2Ñ2°8Ð=MÓNÐNáÐGÔHØà× Ñ Ô"Ø×$Ñ$Ô&ÙØnôð ä—{‘{Ô#=¸xÓHˆHØ'×@Ñ@ÀÓJÐØÐ+Ø×)Ñ)×2Ñ2Ð3CÀXÓNÐNáÐGÔHØäØfó
ð 	
rn   c                ó
  — |j                  «       s|j                  «       sJ ‚|j                  «       r3t        j                  t        |«      }|j                  }|j
                  }n2t        j                  t        |«      }|j                  }|j
                  }d }d }|j                  «       rŒ|j                  «       r|t        j                  t        |«      }t        j                  t        |«      }t        |j                  |j                  «      D ��cg c]  \  }}t        j                  ||«      ‘Œ }	}}�n/|j                  «       r‚t        j                  t        |«      }|j                  |«      }
g }	|}d }|j                  D ]A  }||
u r*t        j                  ||«      }|}|	j                  |«       Œ1|	j                  |«       ŒC n�|j                  «       r‚t        j                  t        |«      }|j                  |«      }g }	|}d }|j                  D ]A  }||u r*t        j                  ||«      }|}|	j                  |«       Œ1|	j                  |«       ŒC nt        d«      ‚ | |j                  |	||||¬«      S c c}}w )NzTAt least one node passed to ForeachKernelSchedulerNode.fuse should be a foreach node)Úuse_custom_partition_algoÚprev_node_1Úprev_node_2Úenable_autotune)r$  rG  rH  ro  r„  r‡  r±  r·   rµ   rr   rz  rÀ   rs  rµ  r  )rq   rr  rv  r„  r‡  r…  r†  r}  r~  Úfused_nodesr‚  r�   Únew_noder�  s                 rl   rr   zForeachKernelSchedulerNode.fuse^	  sZ  € ð ×"Ñ"Ô$¨×(;Ñ(;Ô(=Ð=Ð=Ø×ÑÔ Ü—{‘{Ô#=¸xÓHˆHØ(0×(JÑ(JÐ%Ø&×6Ñ6‰Oä—{‘{Ô#=¸xÓHˆHØ(0×(JÑ(JÐ%Ø&×6Ñ6ˆOØˆØˆà×ÑÔ  X×%8Ñ%8Ô%:Ü—{‘{Ô#=¸xÓHˆHÜ—{‘{Ô#=¸xÓHˆHô   §¡°·±ÓA÷á�A�qô #×'Ñ'¨¨1Õ-ðˆKó ð × Ñ Ô"Ü—{‘{Ô#=¸xÓHˆHØ'×@Ñ@ÀÓJÐØˆKØ"ˆKØˆKØ Ÿ™ò -�ØÐ+Ñ+Ü1×6Ñ6°t¸XÓF�HØ"*�KØ×&Ñ& xÕ0à×&Ñ& tÕ,ñ-ð × Ñ Ô"Ü—{‘{Ô#=¸xÓHˆHØ'×@Ñ@ÀÓJÐØˆKØ"ˆKØˆKà Ÿ™ò -�ØÐ+Ñ+Ü1×6Ñ6°xÀÓF�HØ"*�KØ×&Ñ& xÕ0à×&Ñ& tÕ,ñ-ô !Øfóð ñ Ø×ÑØØ&?Ø#Ø#Ø+ô
ð 	
ùóKs   Ä I?c                ón  •‡ — i ‰ _         i ‰ _        |�|€qt        ‰‰ �  ||«       |D ]Z  }|j                  j
                  D ]  }|‰ j                   |j                  <   Œ |j                  «       D ]  }	|‰ j                  |	<   Œ Œ\ �n<|‰ _        |‰ _	        d ‰ _
        g ‰ _        ‰ j                  t        j                  j                  |j                  |j                  g«      «       t!        ˆ fd„t!        j"                  |j$                  |j$                  «      D «       «      ‰ j                  j&                  z
  ‰ _        t)        |j*                  |j*                  g«      ‰ _        t-        |j.                  |j.                  g«      ‰ _        t)        |j0                  |j0                  «      ‰ _        t-        |j2                  |j2                  «      ‰ _        |j5                  «       rt7        |t8        «      sJ ‚||}}
nt7        |t8        «      sJ ‚||}}
|
j:                  ‰ _        ‰ j:                  j=                  |j:                  «       |
j                  ‰ _        |j                  «       D ]  }	|‰ j                  |	<   Œ ‰ j                  D ���ci c]'  }|j>                  jA                  «       D ]  \  }}||“Œ
 Œ) c}}}‰ _        |‰ _!        |d   jE                  «       }|sJ ‚|tG        jH                  d«      fff‰ _%        t!        tL        jN                  jP                     «       ‰ _)        |‰ _*        y c c}}}w )Nc              3  óZ   •K  — | ]"  }|j                   ‰j                  «       vr|–— Œ$ y ­wrw   rÜ  rã  s     €rl   r”   z6ForeachKernelSchedulerNode.__init__.<locals>.<genexpr>Á	  s0   øè ø€ ò àð —x‘x t×'<Ñ'<Ó'>Ñ>ô	 ñùrÞ  r   Úcombo_kernel)+rq  ru  r>  rx  r°   r±   r²   rà   r  r·   r�   r  r·  r*   rß  rà  r   rá  rq  r�  r‰  rc  r¶  rd  ra  rb  r$  rŒ   ro  rß   r  rl  Úitemsr„  rÞ   rä   ÚExprr¡   rŸ  ÚfxÚNoder_  r‡  )rk   r  r·   r„  r…  r†  r‡  r�   rH  r²   Úforeach_noderê   rø  rÉ  Úvr  rG  s   `               €rl   rx  z#ForeachKernelSchedulerNode.__init__Ÿ	  s  ù€ ð ˆÔØˆÔàÐ +Ð"5Ü‰GÑ˜Y¨Ô/àò 3�Ø ×,Ñ,×2Ñ2ò 8�DØ37�D×%Ñ% d§i¡iÒ0ð8ð !×4Ñ4Ó6ò 3�DØ.2�D×%Ñ% dÒ+ñ3ò	3ð 'ˆDŒNØ ˆDŒKØˆDŒIØ)+ˆDŒJà× Ñ Ü×'Ñ'×2Ñ2Ø ×,Ñ,¨k×.EÑ.EÐFóôô ó ä)×/Ñ/Ø#×6Ñ6¸×8VÑ8Vó ôó ð ×"Ñ"×)Ñ)ñ*ð Ô#ô ! +×"7Ñ"7¸×9NÑ9NÐ!OÓPˆDŒNÜ  +×"7Ñ"7¸×9NÑ9NÐ!OÓPˆDŒNÜ&)Ø×.Ñ.°×0NÑ0Nó'ˆDÔ#ô '*Ø×.Ñ.°×0NÑ0Nó'ˆDÔ#ð ×%Ñ%Ô'Ü! +Ô/IÔJÐJÐJØ+6¸˜j‘ä! +Ô/IÔJÐJÐJØ+6¸˜j�à)×3Ñ3ˆDŒNØ�N‰N×!Ñ! *×"6Ñ"6Ô7à ,× 9Ñ 9ˆDÔØ"×6Ñ6Ó8ò 5�Ø*4�×!Ñ! $Ò'ð5ð #'§+¡+÷@ð @Ø¸%×:OÑ:O×:UÑ:UÓ:Wò@Ù26°!°Q��1‘ð@Øô@ˆDÔ ð *CˆÔ&Ø˜‘×%Ñ%Ó'ˆÙˆˆvØ¤§
¡
¨>Ó :Ð<Ð>Ð?ˆŒ
Ü!¤%§(¡(§-¡-Ñ0Ó2ˆŒØ.ˆÕùô@s   Ê,L0c           
     ó  — |D �cg c]  }t        |t        «      sŒ|‘Œ }}|rSt        j                  dt	        |«      |D �cg c])  }|j
                  €Œ|j
                  j                  «       ‘Œ+ c}«       |D �cg c]  }t        |t        «      sŒ|‘Œ }}|rt        j                  dt	        |«      «       |D �cg c]  }t        |t        «      sŒ|‘Œ }}|rt        j                  dt	        |«      «       |D �cg c]$  }t        |t        t        t        t        f«      s|‘Œ& }}|D �cg c]  }t        |t        «      sŒ|‘Œ }}|rt        j                  dt	        |«      «       |D �cg c]  }t        |t        «      rŒ|‘Œ }}|D �cg c]  }|j                  «       sŒ|‘Œ }	}|	r t        j                  dt	        |	«      |	«       |D �cg c]	  }||	vsŒ|‘Œ }}t        j                  ra|D �cg c]  }|j                  «       sŒ|‘Œ }
}|
rt        j                  dt	        |
«      «       |D �cg c]  }|j                  «       rŒ|‘Œ }}|S c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w )Nz/ComboKernels: %d external nodes are filtered %sz+ComboKernels: %d grouped nodes are filteredz;ComboKernels: %d FusedMixOrderReductions nodes are filteredz+ComboKernels: %d foreach nodes are filteredz0ComboKernels: %d template nodes are filtered: %szCComboKernels: %d reduction nodes are filtered (pointwise_only mode))rŒ   r�  r“  r  r©   r�   rW  rì  r8  rA  ro  r   r(   Úcombo_kernels_pointwise_onlyrŽ   )rq   rÒ  rJ  Úexternr�   ÚgroupedÚ	mix_orderÚfiltered_nodesÚforeach_nodesÚtemplate_nodesÚreduction_nodess              rl   Úcombinable_nodesz+ForeachKernelSchedulerNode.combinable_nodesí	  sd  € ð #ÖO˜¤j°Ô4MÕ&N’!ÐOˆÐOÙÜ�I‰IØAÜ�F“Ø5;ÖU¨T¸t¿y¹yÑ?T�—‘×&Ñ&Õ(ÒUôð
 $ÖK˜¤z°!Ô5IÕ'J’1ÐKˆÐKÙÜ�I‰IØ=Ü�G“ôð !&ÖP˜1¬°AÔ7NÕ)O’QÐPˆ	ÐPÙÜ�I‰IØMÜ�I“ôð ö
àÜØä*Ü-Ü(Ü+ð	ôò ð
ˆð 
ð &ö
Ø¬°AÔ7QÕ)RŠAð
ˆð 
ñ Ü�I‰IÐCÄSÈÓEWÔXà%ö
Ø¬Z¸Ô;UÕ-VŠAð
ˆð 
ð &4ÖG °q·}±}µš!ÐGˆÐGÙÜ�I‰IØBÜ�NÓ#Øôð
 &4ÖO °qÀÒ7Nš!ÐOˆÐOô ×.Ò.Ø*8ÖM Q¸A¿N¹NÕ<LšqÐMˆOÐMÙÜ—	‘	ØYÜ˜Ó(ôð *8ÖP A¸q¿~¹~Õ?OšaÐPˆNÐPàÐùòy Pùò
 VùâKùò Qùò
ùò
ùò

ùò Hùò Pùò Nùò Qs|   …I›IÁIÁIÁ:IÂIÂ;I ÃI Ã<)I%Ä+I*ÅI*Å,I/ÆI/ÆI4Æ"I4Ç	I9ÇI9Ç2I>ÈI>È3JÉ	Jc                ó  — | j                  «       }g }d}t        |D ���cg c]0  }|D ])  }t        |t        «      r|j	                  «       D ]  }|‘Œ Œ+ Œ2 c}}}«      }|D �]  }t        t        «      }	|D ][  }|j                  «       }
|
r|
j                  dk(  s|
j                  dk(  rŒ4|j                  «       |z  rŒH|	|
   j                  |«       Œ] |	j                  «       D ]”  }t        t        «      }|D ]0  }|| j                  j                  |d«         j                  |«       Œ2 |j                  «       D ];  }|j                  t        dt!        |«      |«      D �cg c]
  }||||z    ‘Œ c}«       Œ= Œ– �Œ |S c c}}}w c c}w )zS
        Returns a list of lists of nodes that are to be grouped together.
        é   Úmpsr  r   )Ú_topological_sort_nodesr   rŒ   r8  r  r   rÿ   rÞ   r¶   r¿   rÀ   r¹   Únode_to_streamrÈ  rÞ  r„  r©   )r  Úsorted_nodesÚgrouped_nodesÚmax_num_nodesr¡   r�   r  Úexcluded_buffer_namesrÒ  Údevice_groupsr  Údevice_nodesÚstream_groupsÚstream_nodesrÏ  s                  rl   Ú&_default_group_nodes_for_combo_kernelszAForeachKernelSchedulerNode._default_group_nodes_for_combo_kernels/
  sÎ  € ð !×8Ñ8Ó:ˆØˆØˆä1;ð *÷ð àØ!òð Ü˜dÔ$;Ô<Ø $× 5Ñ 5Ó 7òð
 ò	 ðØðØôó2
Ðð "ó 	ˆEô œDÓ!ð ð ò 3�ØŸ™Ó*�Ù˜vŸ{™{¨eÒ3°v·{±{ÀeÒ7KØð ×)Ñ)Ó+Ð.CÒCØØ˜fÑ%×,Ñ,¨TÕ2ð3ð !.× 4Ñ 4Ó 6ò 
�ÜDOÔPTÓDU�Ø(ò V�DØ! )×":Ñ":×">Ñ">¸tÀQÓ"GÑH×OÑOÐPTÕUðVà$1×$8Ñ$8Ó$:ò �LØ!×(Ñ(ô &+¨1¬c°,Ó.?ÀÓ%Oöà !ð )¨¨Q°Ñ->Ò?òõñò	
ð%	ð: ÐùôKùò@s    5E?Å Fú4Callable[[Scheduler], list[list[BaseSchedulerNode]]]Ú!group_algorithm_for_combo_kernelsc                ó   — | t         _        y rw   ©ro  r¬  )Úcustom_group_algorithms    rl   Ú%set_group_algorithm_for_combo_kernelsz@ForeachKernelSchedulerNode.set_group_algorithm_for_combo_kernelsf
  s   € ð
 #ô 	#ÕDrn   c                ó,   — t         j                  | «      S rw   r®  ©r  s    rl   Úgroup_nodes_for_combo_kernelsz8ForeachKernelSchedulerNode.group_nodes_for_combo_kernelsn
  s   € ô *×KÑKÈIÓVÐVrn   c                ó   — t         ‚rw   r*  rj   s    rl   rÑ  z#ForeachKernelSchedulerNode.mark_runt
  r,  rn   c                ó   — t         ‚rw   r*  rj   s    rl   r;  z"ForeachKernelSchedulerNode.codegenw
  r,  rn   c                 ó   — yrJ  r~   rj   s    rl   r$  z%ForeachKernelSchedulerNode.is_foreachz
  rÇ  rn   c                ó,   — t        | j                  «      S )zeReturns a list of nodes which comprise the combo kernel.
        These nodes may be vertically fused.)rÿ   r·   rj   s    rl   Úget_subkernel_nodesz.ForeachKernelSchedulerNode.get_subkernel_nodes}
  s   € ô �D—K‘KÓ Ð rn   c                ót   — t        t        j                  j                  d„ | j                  D «       «      «      S )zqReturns all nodes contained in this kernel, unpacking fused nodes
        into their constituent scheduler nodes.c              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   )r–   ræ  s     rl   r”   z7ForeachKernelSchedulerNode.get_nodes.<locals>.<genexpr>…
  s   è ø€ Ò1UÀA°!·+±+·-Ñ1Uùrý  )rÿ   rÔ  rÕ  r²  r·   rj   s    rl   r–   z$ForeachKernelSchedulerNode.get_nodes‚
  s(   € ô ”I—O‘O×1Ñ1Ñ1UÈÏÉÔ1UÓUÓVÐVrn   c                ó<   — | j                   d   j                  «       S rÅ   )r·   rù  rj   s    rl   rù  z)ForeachKernelSchedulerNode.get_first_name‡
  s   € Ø�{‰{˜1‰~×,Ñ,Ó.Ð.rn   c                óŠ   — t        | || j                  j                  «       | j                  D ]  }|j	                  |«       Œ y rw   )rò  r  r@  r·   rô  )rk   ró  r�   s      rl   rô  z/ForeachKernelSchedulerNode.prune_redundant_depsŠ
  s=   € ô 	˜dÐ$6¸¿¹×8RÑ8RÔSà—K‘Kò 	:ˆDØ×%Ñ%Ð&8Õ9ñ	:rn   )rr  r_   r†   r  )rv  r_   r†   r  ©rr  r_   rv  r_   r†   rv   )rr  r_   rv  r_   r†   ro  )NNF)r  r  r·   ræ  r„  rv   r…  r  r†  r  r‡  rv   r†   rU  ©rÒ  ræ  r†   ræ  )r  r  r†   úlist[list[BaseSchedulerNode]])r¯  r«  r†   rU  rT  rV  ©r†   ræ  rß  rR  rÝ  )ry   rz   r{   r  rs  rz  r}   rî   rr   rx  rœ  r  rª  r¬  r|   r°  r³  rÑ  r;  r$  r¸  r–   rù  rô  rS  rT  s   @rl   ro  ro  	  s¯  ø… ñð
Ø)ðà	!óðØ)ðà	!óð& ò,
ó ð,
ð\ ð>
Ø(ð>
Ø4Eð>
à	#ò>
ó ð>
ðJ 15Ø04Ø %ðL/àðL/ð (ðL/ð $(ð	L/ð
 .ðL/ð .ðL/ð ðL/ð 
õL/ð\ ð?Ø+ð?à	 ò?ó ð?ðB ð0Øð0à	&ò0ó ð0ðh 	/ð &ð (ó /ð ð
Ø Tð
à	ò
ó ð
ð ðWØðWà	&òWó ðWó
"ó"óó!ó
Wó
/ð:Ø">ð:à	÷:rn   ro  c                  óÌ   ‡ — e Zd ZU dZded<   edd„«       Z	 d	 	 	 	 	 	 	 dˆ fd„Zdd„Zdd„Z	e
dd„«       Zdd	„Ze
dd
„«       Zdd„Ze
dd„«       Zdd„Zdd„Zedd„«       Zˆ xZS )rì  aC  
    This is a "fake" scheduler node that represents a group of scheduler nodes
    that are meant to be *grouped* together (it does not allow another node to be scheduled
    in between its constituent nodes, nor does it allow another node to fuse into any of its constituent nodes).
    The way it does this is by maintaining its unmet dependencies as the union of its constituent nodes.
    Fusion will still happen among the nodes within each GroupedSchedulerNode.
    At codegen time, this scheduler node will be unpacked and codegen is called on each constituent node.
    ræ  r·   c                óæ   ‡— |d   j                   Št        ˆfd„|D «       «      sJ ‚ | ‰|«      }|D ]  }|‰j                  |j                  «       <   Œ! |‰j                  |j                  «       <   |S )Nr   c              3  ó:   •K  — | ]  }|j                   ‰u –— Œ y ­wrw   r²  )r’   r�   r  s     €rl   r”   z.GroupedSchedulerNode.create.<locals>.<genexpr>¢
  s   øè ø€ ÒB°4�4—>‘> YÔ.ÑBùó   ƒ)r  r•   ró  r  )rq   r·   Úgrouped_snoderø  r  s       @rl   ÚcreatezGroupedSchedulerNode.createŸ
  sy   ø€ à˜1‘I×'Ñ'ˆ	ÜÓB¸6ÔBÔBÐBÐBÙ˜I vÓ.ˆØò 	KˆEØ=JˆI×(Ñ(¨¯©Ó)9Ò:ð	KàANˆ	×$Ñ$ ]×%;Ñ%;Ó%=Ñ>ØÐrn   c                óL   •— t         ‰| �  |«       t        | ||«       || _        y rw   )r>  rx  rí  Útemp_grouping)rk   r  r·   rÈ  rG  s       €rl   rx  zGroupedSchedulerNode.__init__©
  s(   ø€ ô 	‰Ñ˜Ô#Ü˜˜i¨Ô0ð +ˆÕrn   c                ó6  — | j                   r| j                  S | j                  D ])  }|| j                  j                  |j	                  «       <   Œ+ | j                  j                  | j	                  «       = | j                  j                  | j                  «      S )z�
        Do fusion among nodes within this GroupedSchedulerNode,
        and then unpack this GroupedSchedulerNode into regular nodes.
        )rÈ  r·   r  ró  r  Ú
fuse_nodes)rk   rø  s     rl   ÚunpackzGroupedSchedulerNode.unpack¸
  sx   € ð
 ×ÒØ—;‘;Ðà—[‘[ò 	HˆEØBGˆD�N‰N×-Ñ-¨e¯n©nÓ.>Ò?ð	Hà�N‰N×-Ñ-¨d¯m©m«oÐ>Ø�~‰~×(Ñ(¨¯©Ó5Ð5rn   c                óŽ   — | j                  | j                  j                  |«      «       | j                  j	                  |«       y rw   )r·  r°   r¼  rq  rÜ  )rk   Úfake_deps     rl   r¾  z!GroupedSchedulerNode.add_fake_depÅ
  s5   € Ø×Ñ˜T×-Ñ-×7Ñ7¸ÓAÔBØ×Ñ×#Ñ# HÕ-rn   c                óz   — dj                  | j                  D �cg c]  }|j                  «       ‘Œ c}«      S c c}w r  r  r  s     rl   r  zGroupedSchedulerNode.get_nameÉ
  r  r  c                ó<   — | j                   d   j                  «       S rÅ   r  rj   s    rl   rù  z#GroupedSchedulerNode.get_first_nameÍ
  r	  rn   c                ó|   — t        j                  | j                  D �cg c]  }|j                  «       ‘Œ c}Ž S c c}w rw   r  r  s     rl   r  z%GroupedSchedulerNode.get_buffer_namesÐ
  r  r  c                ój   — g }| j                   D ]!  }|j                  |j                  «       «       Œ# |S rw   r  r  s      rl   rz  z GroupedSchedulerNode.get_outputsÔ
  r  rn   c                óŽ   — t        t        d d„ | j                  «       D «       «      «      }t        |«      dk(  ry t	        |«      }|S )Nc              3  ó|   K  — | ]4  }|j                  «       s|j                  «       r|j                  «       –— Œ6 y ­wrw   rõ  rü  s     rl   r”   z6GroupedSchedulerNode.estimate_flops.<locals>.<genexpr>à
  rö  r÷  r   rø  rú  s      rl   r£  z#GroupedSchedulerNode.estimate_flopsÚ
  rü  rn   c                ó   — | j                   S rw   r  rj   s    rl   r–   zGroupedSchedulerNode.get_nodesì
  rŸ  rn   c                óX   — | j                   r| j                   d   j                  «       S d S rÅ   )r·   rÞ   rj   s    rl   rÞ   zGroupedSchedulerNode.get_deviceï
  s$   € Ø.2¯kªkˆt�{‰{˜1‰~×(Ñ(Ó*ÐC¸tÐCrn   c                 ó   — yr¬  r~   )rq   rr  rv  s      rl   rî   zGroupedSchedulerNode.can_fuseò
  r°  rn   )r·   ræ  r†   rì  )F)r  r  r·   ræ  rÈ  rv   r†   rU  rÀ  )rÍ  r3   r†   rU  rR  rÜ  r5  râ  rß  rX  r½  )ry   rz   r{   r  r|   r}   rÆ  rx  rË  r¾  rI   r  rù  r  rz  r£  r–   rÞ   rî   rS  rT  s   @rl   rì  rì  “
  sÃ   ø… ñð $Ó#àòó ðð $ð	+àð+ð (ð+ð ð	+ð
 
õ+ó6ó.ð ò=ó ð=ó)ð òNó ðNóð òó ðó"óDð òó ôrn   rì  c           
     ó   ‡ ‡— t         j                  dˆˆ fd„«       }t        t        t	        t        ‰ d   «      «      «      «      }t        |«      dkD  r|D �cg c]  }‰ |   ‘Œ	 c}Š t        j                  r|j                  |¬«       |S c c}w )z�
    A heuristic to decide loop iteration orders.  This has not been well
    tuned and may be something we should autotune.
    c                ót  •— ‰|    dk(  s‰|   dk(  rt        ‰|    dk(  ‰|   dk(  «      S ‰D �cg c]  }t        ||    «      ‘Œ }}‰D �cg c]  }t        ||   «      ‘Œ }}t        d„ t        ||«      D «       «      }t        d„ t        ||«      D «       «      }||kD  ry||kD  ryt        || «      S c c}w c c}w )Nr   c              3  ó:   K  — | ]  \  }}|d k(  xs ||k  –— Œ y­w©r   Nr~   ©r’   Úsl_aÚsl_bs      rl   r”   z5pick_loop_order.<locals>.index_cmp.<locals>.<genexpr>  ó)   è ø€ ò 
Ù)3¨¨tˆD�A‰IÒ$˜ ™Ó$ñ
ùó   ‚c              3  ó:   K  — | ]  \  }}|d k(  xs ||k  –— Œ y­wrÚ  r~   rÛ  s      rl   r”   z5pick_loop_order.<locals>.index_cmp.<locals>.<genexpr>  rÞ  rß  r÷   )rJ   ÚabsrÙ   r±  )	rò  ÚbÚslÚstride_len_aÚstride_len_bÚa_firstÚb_firstr~  Ústride_lengthss	          €€rl   Ú	index_cmpz"pick_loop_order.<locals>.index_cmp  sÛ   ø€ à�‰8�qŠ=˜E !™H¨šMä�u˜Q‘x 1‘} e¨A¡h°!¡mÓ4Ð4ð .<Ö< rœ˜B˜q™E�
Ð<ˆÐ<Ø-;Ö< rœ˜B˜q™E�
Ð<ˆÐ<ô ñ 
Ü7:¸<ÈÓ7Vô
ó 
ˆô ñ 
Ü7:¸<ÈÓ7Vô
ó 
ˆð �WÒØØ�WÒØô �1�a‹yÐùò# =ùÚ<s   ®B0Á	B5r   r$  )rò  r  râ  r  r†   r  )	Ú	functoolsÚ
cmp_to_keyrÿ   r«   r„  r©   r(   Úpick_loop_ordersÚsort)rè  r~  Úpriority_idxré  ÚorderÚpis   ``    rl   Úpick_loop_orderrñ  ø
  s�   ù€ ô ×Ñõó ðô4 ”œ%¤ N°1Ñ$5Ó 6Ó7Ó8Ó9€EÜ
ˆ<Ó˜1Òà7CÖD°˜.¨Ó,ÒDˆÜ×ÒØ�
‰
�yˆ
Ô!Ø€Lùò Es   ÁBc                óÄ  — |j                  «       }| j                  «       }t        |t        «      rt        |t        «      sJ ‚|j                  «       }| j                  «       }t        |t        «      rt        |t        «      sJ ‚t        j
                  j                  |= ||_        t        j
                  j                  |= ||_	        t        j
                  j                  j                  | «      }t        j
                  j                  j                  |«       |t        j
                  j                  |<   |t        j
                  j                  |<   t        j
                  j                  j                  | «      }t        j
                  j                  j                  |«       |t        j
                  j                  |<   |t        j
                  j                  |<   y rw   )r  rŒ   r  rö  rZ   rœ   rÝ  r²   Ú
name_to_opÚoperation_nameÚbuffersr³   r\  Ú
operations)Ú	orig_noder‰  Úreplaced_buf_nameÚorig_buf_nameÚreplaced_op_nameÚorig_op_nameÚorigs          rl   Ú_replace_operation_bufferrý  &  sU  € ð !×)Ñ)Ó+ÐØ×&Ñ&Ó(€MÜ�m¤SÔ)¬jÐ9JÌCÔ.PÐPÐPà×2Ñ2Ó4ÐØ×/Ñ/Ó1€LÜ�l¤CÔ(¬ZÐ8HÌ#Ô-NÐNÐNä	�‰×ÑÐ0Ð1Ø!€H„Mä	�‰×ÑÐ+Ð,Ø*€HÔä�7‰7�?‰?× Ñ  Ó+€DÜ‡G�G‡O�O×Ñ˜8Ô$Ø$„A‡G�G‡O�O�DÑØ,4„A‡G�G×Ñ˜=Ñ)ä�7‰7×Ñ×#Ñ# IÓ.€DÜ‡G�G×Ñ×Ñ˜hÔ'Ø'„A‡G�G×Ñ�tÑØ'/„A‡G�G×Ñ�|Ò$rn   c                óp   — | j                  «       }|j                  «       }||z
  }||z  }|d|z   z  }||z  S r[  )rn  rl  )r�   r‚   Úepilogue_runtimeÚtemplate_write_bytesÚepilogue_read_bytesÚextra_bytesÚextra_bytes_ratioÚextra_memory_ratios           rl   Ú _estimate_fused_epilogue_runtimer  B  sX   € Ø ×7Ñ7Ó9ÐØ×5Ñ5Ó7ÐØ%Ð(<Ñ<€Kð $Ð&:Ñ:Ðð +¨aÐ2CÑ.CÑDÐØÐ 0Ñ0Ð0rn   c                óŠ   — |dk\  ry|j                   }|€y|sJ ‚||j                  xs dz  }| |z  }||z  }||z  }	||z  }
|	|
fS )Nrž  )r   r÷   )r   r   é    )Úregs_per_multiprocessorÚ	warp_size)Úunfused_n_regsÚfused_n_regsÚfused_n_spillsÚ	num_warpsÚdevice_propsÚregs_per_smÚthreads_per_blockÚregs_per_block_unfusedÚregs_per_block_fusedÚblocks_unfusedÚblocks_fuseds              rl   Ú"_occupancy_before_and_after_fusionr  O  s|   € ð ˜ÒØð ×6Ñ6€KØÐØáÐˆ9Ø! \×%;Ñ%;Ò%A¸rÑBÐà+Ð.?Ñ?ÐØ'Ð*;Ñ;Ðà Ð$:Ñ:€NØÐ"6Ñ6€Là˜<Ð'Ð'rn   c                ó€   — d}d}t        |||||«      \  }	}
|d| z  kD  xr |
dkD  }|
dk7  xr |
|k\  xs |
|	z  |kD  xs |S )zE
    Determine whether to fuse an epilogue into a GEMM template.
    r&  g      à?r¨   r   r÷   )r  )Úms1Úms2r
  r  r  r  r  ÚMIN_ACCEPTED_OCCUPANCYÚREGRESSED_OCCUPANCY_RATIOr  r  Ú,epilogue_dominated_with_sufficient_occupancys               rl   Ú_fuse_epiloguer  j  s€   € ð ÐØ #Ðô $FØ˜ n°iÀó$Ñ €N�Lð 47¸¸S¹±=Ò3UÀ\ÐTUÑEUÐ0ð
 ˜2Ñò ØÐ.Ñ.ò 	8Ø˜.Ñ(Ð+DÑDò	8à7ðrn   c                  óT   — e Zd ZU ded<   dZded<   dZded<   dd„Zdd„Zdd	„Zdd
„Z	y)r]  úBaseSchedulerNode | OutputNoder�   Frv   r(  Úis_weakc                óv   — t        | j                  j                  «       | j                  | j                  f«      S rw   )r   r�   r  r(  r  rj   s    rl   r!  zNodeUser.__hash__“  s+   € Ü�T—Y‘Y×'Ñ'Ó)¨4×+;Ñ+;¸T¿\¹\ÐJÓKÐKrn   c                óÔ   — t        |t        «      xrW | j                  «       |j                  «       k(  xr4 | j                  |j                  k(  xr | j                  |j                  k(  S rw   )rŒ   r]  r  r(  r  rM  s     rl   Ú__eq__zNodeUser.__eq__–  s[   € ä�uœhÓ'ò .Ø—‘“ 5§>¡>Ó#3Ñ3ò.à× Ñ  E×$5Ñ$5Ñ5ò.ð —‘ §¡Ñ-ð		
rn   c                ó6   — | j                   j                  «       S rw   r1  rj   s    rl   r  zNodeUser.get_namež  r2  rn   c                óÄ   — | j                   |j                   u sJ ‚t        | j                   | j                  xr |j                  | j                  xr |j                  «      S rw   )r�   r]  r(  r  rM  s     rl   rL  zNodeUser.merge¡  sP   € Ø�y‰y˜EŸJ™JÑ&Ð&Ð&ÜØ�I‰IØ×ÑÒ2 ×!2Ñ!2Ø�L‰LÒ*˜UŸ]™]ó
ð 	
rn   NrS  )rN  Úobjectr†   rv   rR  )rN  r]  r†   r]  )
ry   rz   r{   r|   r(  r  r!  r"  r  rL  r~   rn   rl   r]  r]  Š  s3   … à
(Ó(Ø€K�Óð €GˆTÓóLó
ó$ô
rn   r]  c                 ó"   — t         j                  S rw   )r(   r  r~   rn   rl   Ú*used_non_deterministic_runtime_estimationsr'  ­  s   € Ü×3Ñ3Ð3rn   c                ó¦  — t        «       }| j                  «       }t        |t        j                  «      r�|j                  t        |j                  «      t        |j                  «      z  t        |j                  «      z  «       t        |t        j                  «      r$|j                  t        |j                  «      «       |S |�
J d|› �«       ‚|S )z=Get free symbols from a node's layout (size, stride, offset).z*Expect layout to be None but found layout=)r   Úmaybe_get_layoutrŒ   r+   ÚLayoutr  r"   r¸   ÚstrideÚoffsetrC  Úget_layout_symintsrX  )r�   Úfree_symbol_usesr'  s      rl   r-  r-  ±  s³   € ä1;³ÐØ×"Ñ"Ó$€FÜ�&œ"Ÿ)™)Ô$Ø×ÑÜ˜Ÿ™Ó%Ü˜6Ÿ=™=Ó)ñ*ä˜6Ÿ=™=Ó)ñ*ô	
ô
 �fœb×;Ñ;Ô<à×#Ñ#Ô$6°v·}±}Ó$EÔFð Ðð ˆ~ÐTÐ!KÈFÈ8ÐTÓTˆ~ØÐrn   c                ó"  — t        | t        «      r( t        «       j                  d„ | j                  D «       Ž S | j
                  €J ‚| j
                  j                  «       } |j                  d„ | j
                  j                  «       D «       Ž  |S )z‡
    Gets symbols used in a scheduler node, including free symbols from
    the node's operations and layout symints from outputs.
    c              3  ó2   K  — | ]  }t        |«      –— Œ y ­wrw   ©Úget_scheduler_node_symbol_uses)r’   rø  s     rl   r”   z1get_scheduler_node_symbol_uses.<locals>.<genexpr>Ì  s   è ø€ ÒM¸Ô,¨U×3ÑMùó   ‚c              3  ó2   K  — | ]  }t        |«      –— Œ y ­wrw   )r-  )r’   Úir_nodes     rl   r”   z1get_scheduler_node_symbol_uses.<locals>.<genexpr>Ñ  s   è ø€ Ò	M¨'Ô
˜W×
%Ñ	Mùr3  )	rŒ   rµ   r   rá  r·   r�   Úget_free_symbol_usesr  rz  )r�   r.  s     rl   r2  r2  Ã  s…   € ô �$Ô*Ô+Ø!Œz‹|×!Ñ!ÙMÀÇÁÔMð
ð 	
ð �9‰9Ð Ð Ð Ø—y‘y×5Ñ5Ó7ÐØÐ×ÑÙ	M°T·Y±Y×5JÑ5JÓ5LÔ	Mñð Ðrn   c                óv   — | j                  «       }|�|j                  �|j                  S t        j                  S ©z4Check per-template flag, fall back to global config.)rÆ  Úallow_epilogue_fusionr(   Úepilogue_fusion©rÕ  Útbs     rl   Ú_is_epilogue_fusion_enabledr=  Ö  ó8   € à	×	(Ñ	(Ó	*€BØ	€~˜"×2Ñ2Ð>Ø×'Ñ'Ð'Ü×!Ñ!Ð!rn   c                óv   — | j                  «       }|�|j                  �|j                  S t        j                  S r8  )rÆ  Úallow_prologue_fusionr(   Úprologue_fusionr;  s     rl   Ú_is_prologue_fusion_enabledrB  Þ  r>  rn   c                ób   — | j                  «       xr |j                  «        xr t        | «      S rw   )r   r=  r„   s     rl   Úis_epilogue_fusionrD  æ  ó4   € à×ÑÓò 	/Ø×!Ñ!Ó#Ð#ò	/ä'¨Ó.ðrn   c                ób   — |j                  «       xr | j                  «        xr t        |«      S rw   )r   rB  r„   s     rl   Úis_prologue_fusionrG  î  rE  rn   c                ó6   — t        | |«      xs t        | |«      S rw   )rD  rG  r„   s     rl   Úis_template_fusionrI  ö  s   € Ü˜e UÓ+ÒOÔ/AÀ%ÈÓ/OÐOrn   c                ó"   — t        | |«      r|S | S rw   )rD  r„   s     rl   Útemplate_fusion_pw_noderK  ú  s   € Ü& u¨eÔ4ˆ5Ð?¸%Ð?rn   c                  ó  ‡ — e Zd ZdZdjd„Zdjˆ fd„Zdkd„Zdld„Zdmd„Zdnd„Z	dod„Z
edpd	„«       Zej                  dqd
„«       Zdld„Zdrd„Zdsd„Zdld„Zdld„Zdld„Zdld„Zdtd„Z	 	 	 	 dud„Zdld„Zdvd„Zdwd„Zdld„Zdld„Zdld„Zdud„Zdld„Z	 	 	 	 dxd„Z	 dy	 	 	 	 	 	 	 dzd„Z 	 	 	 	 	 	 d{d„Z!	 	 	 	 d|d„Z"dld „Z#	 	 	 	 	 	 	 	 	 	 d}d!„Z$d~d"„Z%	 dy	 	 	 	 	 dd#„Z&	 	 	 	 	 	 d€d$„Z'd�d%„Z(	 	 	 	 	 	 	 	 d‚d&„Z)	 	 	 	 	 	 	 	 dƒd'„Z*	 	 	 	 	 	 d„d(„Z+	 	 	 	 	 	 	 	 	 	 d…d)„Z,	 	 	 	 d†d*„Z-	 	 	 	 d‡d+„Z.	 	 	 	 	 	 dˆd,„Z/dyd‰d-„Z0dŠd.„Z1	 	 	 	 	 	 d‹d/„Z2	 	 	 	 	 	 dŒd0„Z3	 	 	 	 	 	 dŒd1„Z4	 	 	 	 	 	 	 	 d�d2„Z5	 	 	 	 	 	 dŒd3„Z6	 	 	 	 	 	 	 	 dŽd4„Z7	 	 	 	 	 	 d�d5„Z8	 	 	 	 	 	 d�d6„Z9d�d7„Z:	 	 	 	 	 	 	 	 d‘d8„Z;	 	 	 	 	 	 d’d9„Z<	 	 d“	 	 	 	 	 	 	 	 	 d”d:„Z=	 	 	 	 	 	 dŒd;„Z>	 	 	 	 	 	 	 	 d•d<„Z?d–d=„Z@	 	 	 	 	 	 	 	 d—d>„ZAd˜d™d?„ZB	 	 	 dš	 	 	 	 	 	 	 	 	 	 	 d›d@„ZC	 	 	 	 	 	 dŒdA„ZD	 	 	 	 	 	 d�dB„ZE	 	 	 	 dœdC„ZF	 	 	 	 d�dD„ZGdldE„ZHdldF„ZIdldG„ZJ	 	 	 	 dždH„ZKdŸdI„ZLd dJ„ZMd¡dK„ZN	 	 	 	 	 	 d¢dL„ZOd£dM„ZPeQd¤dN„«       ZR	 	 	 	 d£dO„ZS	 	 d¥dP„ZT	 	 	 	 d¦dQ„ZU	 	 	 	 	 	 d§dR„ZV	 	 	 	 	 	 d¨dS„ZW	 	 	 	 d©dT„ZX	 	 	 	 dudU„ZY	 	 	 	 dudV„ZZ	 	 	 	 dudW„Z[	 	 dªdX„Z\	 	 	 	 	 	 d«dY„Z]d¡dZ„Z^dld[„Z_	 	 	 	 	 	 d¬d\„Z`	 	 	 	 	 	 d­d]„Za	 	 	 	 	 	 d«d^„Zbdld_„ZcdŠd`„Zd	 	 	 	 d®da„Zed¯db„Zfd°dc„Zgdldd„Zhed±de„«       Zied²df„«       Zjd¡dg„Zkdldh„Zld¡di„Zmˆ xZnS )³r  z�
    A Scheduler is a graph of BaseSchedulerNodes. It is responsible for
    optimizations such as fusion, reorder, and graph partition.
    c                óf   — t        d«      5  | j                  |«       d d d «       y # 1 sw Y   y xY w)NzScheduler.__init__)r   Ú_init©rk   rÒ  s     rl   rx  zScheduler.__init__  s,   € ÜÐ.Ó/ñ 	Ø�J‰J�uÔ÷	÷ 	ñ 	ús   Œ'§0c           
     ó  •‡ — t         ‰‰ �  «        ‰ t        j                  _        i ‰ _        t        t        «      ‰ _        t        j                  «       ‰ _        t        «       ‰ _        t        g t        j                  j                  j                  «       ¢t        j                  j                   j                  «       ¢t        j                  j"                  j                  «       ¢«      ‰ _        |D �cg c]  }‰ j'                  |«      ‘Œ c}‰ _        d ‰ _        d ‰ _        ‰ j/                  «        ‰ j$                  j1                  t        j                  j                   j                  «       «       ‰ j(                  D ]  }|j3                  «        Œ d ‰ _        ‰ j7                  «       ‰ _        ‰ j(                  D �ci c]  }|j;                  «       |“Œ c}‰ _        ‰ j(                  D ��ci c](  }|j?                  «       D ]  }|j;                  «       |“Œ Œ* c}}‰ _         ‰ j<                  jC                  «       ‰ _"        i ‰ _#        i ‰ _$        t        «       ‰ _%        tM        jN                  ‰ j(                  ‰ j@                  ‰ jD                  «      ‰ _        ‰ jQ                  «        ‰ jS                  ‰ j(                  «      ‰ _        ‰ jU                  «        ‰ j(                  D �ci c]  }|j;                  «       |“Œ c}‰ _"        ‰ jW                  «        ‰ jY                  «        tZ        xj\                  t_        ‰ j(                  «      z  c_.        ddl0m1}m2}  |‰ j(                  «       t_        ‰ j(                  «      ‰ _3        ‰ ji                  «        ‰ jS                  ‰ j(                  «      ‰ _        t        tj        tl        tl        f      «       ‰ _7        tp        jr                  �$tq        jr                  ‰ j(                  «      ‰ _        tp        jt                  r'ddl;m<} |j{                  ‰ «       ‰ jW                  «        i ‰ _>        i ‰ _?        d‰ _@        i ‰ _A        ‰ j…                  «        ‰ j‡                  ‰ j(                  «      ‰ _        tp        jˆ                  �$tq        jˆ                  ‰ j(                  «      ‰ _        t‹        d„ ‰ j(                  D «       «      r‰ jU                  «        ‰ j�                  «        ‰ j�                  «        tp        j�                  stp        j’                  r<t•        «       r2t–        j˜                  jš                  jœ                  jŸ                  «        tp        j                   r)t£        ddd¬	«      5  ‰ j¥                  d ¬
«       d d d «       ‰ j§                  «        tp        j¨                  r‰ddlUmT}  |‰ j(                  ‰ j@                  ‰ jD                  t        t        j                  j                  j                  «       «      t        t        j                  j­                  «       «      «      ‰ _        tp        j®                  �stp        j°                  rütp        j¨                  s#ddlUmY}	  |	‰ j(                  ‰ j@                  «       tµ        «       rvt¶        j¸                  rftp        jº                  st¶        j¼                  rFd}
‰ j(                  D ]  }t¿        |jÀ                  «      sŒd}
 n |
rddl&ma}  |‰ j(                  «       t¶        jÄ                  rddlcmd}  |dd„ ˆ fd„¬«       tM        jÊ                  ‰ j(                  «      ‰ _        ‰ jÍ                  «        tp        jÎ                  rttp        jÐ                  jÒ                  rZtp        jÐ                  jÔ                  r@‰ j×                  ‰ j(                  «      ‰ _        ‰ jÙ                  ‰ j(                  «      ‰ _        ‰ jÛ                  «        t–        j˜                  jp                  jÜ                  jÞ                  r‰ já                  «         |‰ j(                  «       t        jâ                  jå                  ‰ j(                  «       ‰ jç                  «        t        «       ‰ _t        i ‰ _u        d ‰ _v        tï        d«      jñ                  ˆ fd„«       t        «       ‰ _y        y c c}w c c}w c c}}w c c}w # 1 sw Y   �Œ\xY w)Nr   )Úlog_ir_post_fusionÚlog_ir_pre_fusionr   )Údistributed_autotuneFc              3  ó<   K  — | ]  }t        |t        «      –— Œ y ­wrw   )rŒ   rV  rü  s     rl   r”   z"Scheduler._init.<locals>.<genexpr>q  s    è ø€ ò 
àô �tÔA×Bñ
ùrý  z#Scheduler.create_combo_kernel_nodesT©Úlog_pt2_compile_eventÚlog_waitcounter)Únum_ck_nodes)Úreorder_for_peak_memory)Ú1assign_memory_planning_info_for_scheduler_buffers)Ú6align_runtime_estimations_across_all_distributed_ranks)Útrace_structuredÚartifactc                 ó   — dddœS )NÚ#scheduler_nodes_before_comm_overlapÚstring)r²   Úencodingr~   r~   rn   rl   rv  z!Scheduler._init.<locals>.<lambda>¾  s   € Ø EØ$,ñ)€ rn   c            
     óÐ   •— dj                  t        ‰j                  «      D � �cg c]0  \  } }d| › d�|j                  «       z   d|j	                  «       › �z   ‘Œ2 c}} «      S c c}} w )Nz

zsnode[r%  z buffer_names:)rœ  rÑ  rÒ  r/  r  )rÏ  r  rk   s     €rl   rv  z!Scheduler._init.<locals>.<lambda>Â  sl   ø€  v§{¡{ô
 )2°$·*±*Ó(=÷	ñ !%  1ð % Q C q˜MØŸk™k›mñ,à .¨q×/AÑ/AÓ/CÐ.DÐEóFóó(€ ùós   ¥5A"
)Úmetadata_fnÚ
payload_fnÚgraph_statsc                 ó^   •— ‰ j                   ‰ j                  t        ‰ j                  «      dœS )N)Úgraph_idÚnum_nodes_before_fusionÚnum_nodes_after_fusion)Úpost_grad_graph_idÚnum_orig_nodesr©   rÒ  rj   s   €rl   rv  z!Scheduler._init.<locals>.<lambda>î  s'   ø€ Ø ×3Ñ3Ø+/×+>Ñ+>Ü*-¨d¯j©j«/ñ€ rn   )zr>  rx  rZ   rœ   r  ÚbackendsrÐ  Ú_post_grad_graph_counterrj  rÔ  ÚcountÚ_graph_partition_counterr   r>  r–  r   Ú	constantsÚtorchbind_constantsrâ  Úcreate_scheduler_noderÒ  Úprevious_nodeÚcurrent_nodeÚupdate_zero_dim_cpu_tensorr  rÄ  Údefault_device_contextÚget_donated_buffersr?  r  ru  rz  r@  Úcopyró  rÊ  rg  Úseen_template_fusionsr'   Údecide_global_ordering_of_commsr]   Útopological_sort_scheduleÚdead_node_eliminationÚcompute_ancestorsÚcompute_input_distancesr,   Úir_nodes_pre_fusionr©   Útorch._inductor.debugrQ  rR  rk  Úcreate_foreach_nodesrª   r  Úlogged_slow_fusionr(   Ú_pre_fusion_custom_passÚdistributed_max_autotune_gemmr—  rS  Úscheduler¡  Úbuff_to_streamÚ_multi_stream_nodesÚstream_idx_to_user_obj_idxÚ_populate_stream_assignmentsrÊ  Ú_post_fusion_custom_passræ   r”  Úfinalize_multi_template_buffersÚmax_autotune_gemmÚmax_autotuner   rŸ  r   Úselect_algorithmÚPrecompileThreadPoolÚshutdown_instanceÚcombo_kernelsr   Úcreate_combo_kernel_nodesÚ_enforce_conditional_orderingrY  ÚmemoryÚget_output_namesÚdeterministicÚ reorder_for_compute_comm_overlaprZ  r'  r)   Ú6runtime_estimations_align_across_all_distributed_ranksr  r«  rS   r�   r[  Úreorder_sink_verbose_loggingÚtorch._loggingr\  Ú$reorder_compute_and_comm_for_overlapÚprocess_grouped_nodesÚgraph_partitionrÒ   r^   Ú%reorder_for_reducing_graph_partitionsÚ&maybe_reorder_for_minimizing_partitionÚ,reorder_for_partition_with_simple_dependencyÚcompute_last_usageÚtest_configsÚtrack_memory_lifecycleÚinsert_memory_check_nodesr  Úgraph_diagramÚdebug_draw_graphÚbuffer_names_to_freeÚorigin_to_indexÚ_current_stream_ctxr   Úadd_rowÚremoved_ops)rk   rÒ  r  r�   rº   rQ  rR  rS  rY  rZ  Úhas_collectivesr[  r\  rG  s   `            €rl   rN  zScheduler._init  sÎ  ù€ Ü‰ÑÔØ Œ�‰ÔØ<>ˆŒÜ"&Ô'?Ó"@ˆÔÜ(1¯©Ó(9ˆÔ%ä5?³\ˆÔ!Ü&0ðÜ—‘×%Ñ%×*Ñ*Ó,ðä—‘×"Ñ"×'Ñ'Ó)ðô —‘×,Ñ,×1Ñ1Ó3ðó'
ˆÔ#ð >CÖC¸�d×0Ñ0°Õ3ÒCˆŒ
Ø7;ˆÔØ6:ˆÔØ×'Ñ'Ô)à×#Ñ#×*Ñ*¬1¯7©7×+<Ñ+<×+AÑ+AÓ+CÔDØ—J‘Jò 	ˆDØ�O‰OÕð	ð <@ˆÔ#ð ×$Ñ$Ó&ð 	Ô#ð &*§Z¡Zö;
Ø !ˆA�J‰J‹L˜!‰Oò;
ˆÔð
 -1¯J©J÷8
Ø$(À$×BRÑBRÓBTò8
Ø;>ˆC�L‰L‹N˜CÑð8
Øó8
ˆÔð AE×@QÑ@Q×@VÑ@VÓ@XˆÔð 35ˆÔð 13ˆÔô ‹Lð 	Ô"ô
 ×:Ñ:Ø�J‰JØ×ÑØ×#Ñ#ó
ˆŒ
ð 	×!Ñ!Ô#Ø×3Ñ3°D·J±JÓ?ˆŒ
Ø×"Ñ"Ô$Ø<@¿J¹JÖ"G°q 1§:¡:£<°¡?Ò"GˆÔØ×ÑÔ Ø×$Ñ$Ô&ô 	×#Ò#¤s¨4¯:©:£Ñ6Õ#ßOá˜$Ÿ*™*Ô%Ü! $§*¡*›oˆÔØ×!Ñ!Ô#Ø×3Ñ3°D·J±JÓ?ˆŒ
Ü",¬U´3¼°8©_Ñ"=Ó"?ˆÔÜ×)Ñ)Ð5Ü×7Ñ7¸¿
¹
ÓCˆDŒJä×/Ò/Ý.à ×)Ñ)¨$Ô/Ø×"Ñ"Ô$ð =?ˆÔØ.0ˆÔØ).ˆÔ à:<ˆÔ'Ø×)Ñ)Ô+à—_‘_ T§Z¡ZÓ0ˆŒ
Ü×*Ñ*Ð6Ü×8Ñ8¸¿¹ÓDˆDŒJäñ 
àŸ
™
ô
ô 
ð ×&Ñ&Ô(à×ÑÔØ×,Ñ,Ô.ä×$Ò$¬×(;Ò(;Ü&Ô(Ü�O‰O×,Ñ,×AÑA×SÑSÔUä×ÒÜØ5Ø&*Ø $ôñ Bð
 ×.Ñ.¸DÐ.ÔA÷Bð 	×*Ñ*Ô,ô ×)Ò)Ý7á0Ø—
‘
Ø× Ñ Ø×'Ñ'Üœ1Ÿ7™7×/Ñ/×4Ñ4Ó6Ó7Üœ1Ÿ7™7×3Ñ3Ó5Ó6óˆDŒJô ×#Ó#¬×(OÒ(OÜ×1Ò1ÝUáAØ—J‘J × 0Ñ 0ôô
 ;Ô<Ü ×WÒWä×<Ò<Ü#×PÒPð #(�Ø ŸJ™Jò �DÜ$ T§Y¡YÕ/Ø*.˜Ùðñ #õñ KÈ4Ï:É:ÔVô ×8Ò8Ý;á Øñ!ó õô ×CÑCÀDÇJÁJÓOˆDŒJØ×"Ñ"Ô$ô ×"Ò"ä—‘×(Ò(ä—‘×CÒCà×DÑDÀTÇZÁZÓPˆDŒJØ×JÑJÈ4Ï:É:ÓVˆDŒJà×ÑÔ!ä�?‰?×!Ñ!×.Ñ.×EÒEØ×*Ñ*Ô,á˜4Ÿ:™:Ô&ä	�‰×Ñ˜dŸj™jÔ)Ø×ÑÔô 6@³\ˆÔ!ð :<ˆÔð GKˆÔ ä˜Ó'×/Ñ/óô	
ô -7«LˆÕùòA Dùò;
ùó8
ùòH #H÷hBñ Bús$   Ã5c$Æ?c)Ç.-c.Ë(c4Öc9ã9dc                ó  — i }t         j                  j                  D ]d  }t        t         j                  j                  |   t        j
                  «      sŒ9t        | t         j                  j                  |   d ¬«      ||<   Œf |S )N)r  )rZ   rœ   Úgraph_inputs_originalrŒ   r+   ÚDonatedBufferr]  )rk   Úname_to_donated_bufr²   s      rl   rw  zScheduler.get_donated_buffersù  sp   € Ø ÐÜ—G‘G×1Ñ1ò 	ˆDÜœ!Ÿ'™'×7Ñ7¸Ñ=¼r×?OÑ?OÕPÜ,BØÜ—G‘G×1Ñ1°$Ñ7Ø $ô-Ð# DÒ)ð	ð #Ð"rn   c                óv  ‡
— ddl mŠ
 i }t        j                  d«      }| j                  D ]‡  }‰
}|j
                  �D|j
                  j                  «       }|�(||vrt        |«      }|||<   || j                  |<   ||   }|| j                  |<   |j                  «       D ]  }|| j                  |<   Œ Œ‰ t        ˆ
fd„| j                  j                  «       D «       «      r¨t        d„ | j                  D «       d«      }|�‰| j                  D ]z  }|j
                  }	|j                  «       �Œ t        |	t         j"                  «      sŒ;t        |	j$                  t         j&                  «      sŒ`t!        j&                  |¬«      |	_        Œ| t        ˆ
fd„| j                  j                  «       D «       «      | _        y)a=  Populate node_to_stream and buff_to_stream from IR node stream_idx.

        Reads the stream_idx field set on IR nodes during lowering to determine
        which stream each scheduler node should run on. This field is propagated
        from 'custom.stream' FX node metadata via IRNode.current_stream_idx().
        r   )ÚDEFAULT_STREAM_IDXNc              3  ó(   •K  — | ]	  }|‰k7  –— Œ y ­wrw   r~   )r’   rt  r²  s     €rl   r”   z9Scheduler._populate_stream_assignments.<locals>.<genexpr>'  s   øè ø€ ÒM¨1ˆqÐ&Õ&ÑMùó   ƒc              3  ó^   K  — | ]%  }|j                  «       €Œ|j                  «       –— Œ' y ­wrw   ©rÞ   r  s     rl   r”   z9Scheduler._populate_stream_assignments.<locals>.<genexpr>)  s!   è ø€ ÒR A°q·|±|³~Ñ7Q�—‘—ÑRùs   ‚-˜-©r  c              3  ó(   •K  — | ]	  }|‰k7  –— Œ y ­wrw   r~   )r’   Ú
stream_idxr²  s     €rl   r”   z9Scheduler._populate_stream_assignments.<locals>.<genexpr>7  s   øè ø€ ò '
àð Ð,Õ,ñ'
ùr´  )Ústream_constantsr²  rÔ  rn  rÒ  r�   Úget_stream_idxrÐ  rˆ  r¡  r  r†  ræ   r¹   rÞ   rŒ   r+   ÚBufferr'  r?   r‡  )rk   Úuser_obj_to_stream_idxÚstream_idx_counterr�   r¹  Úuser_obj_idxÚnew_stream_idxrº   r  r5  r²  s             @rl   r‰  z&Scheduler._populate_stream_assignments  s”  ø€ õ 	9ð 24ÐÜ&Ÿ_™_¨QÓ/Ðà—J‘Jò 	6ˆDØ+ˆJà�y‰yÐ$Ø#Ÿy™y×7Ñ7Ó9�ØÐ+Ø#Ð+AÑAÜ)-Ð.@Ó)A˜Ø?MÐ.¨|Ñ<ØJV˜×7Ñ7¸ÑGØ!7¸Ñ!E�Jà(2ˆD×Ñ Ñ%ð ×,Ñ,Ó.ò 6�Ø+5�×#Ñ# CÒ(ñ6ð!	6ô, ÓM°×0CÑ0C×0JÑ0JÓ0LÔMÔMÜÙR¨¯©ÔRÐTXóˆFð Ð!Ø ŸJ™Jò F�DØ"Ÿi™i�GàŸ™Ó)Ñ1Ü& w´·	±	Õ:Ü& w§~¡~´r·}±}ÕEô *,¯©¸fÔ)E˜�ðFô $'ó '
à"×1Ñ1×8Ñ8Ó:ô'
ó $
ˆÕ rn   c                ó   — | j                   S )z7Check if any nodes are assigned to non-default streams.)r‡  rj   s    rl   Ú_has_multi_stream_nodesz!Scheduler._has_multi_stream_nodes<  s   € à×'Ñ'Ð'rn   c                ó¦   — | j                   j                  ||«      }| j                  j                  || j                  j                  |d«      «      S )zAReturn the stream index for a buffer, resolving mutation renames.r   )rg  rÈ  r†  )rk   r  Úreals      rl   Úget_buf_streamzScheduler.get_buf_stream@  sF   € à×$Ñ$×(Ñ(¨°8Ó<ˆØ×"Ñ"×&Ñ& t¨T×-@Ñ-@×-DÑ-DÀXÈqÓ-QÓRÐRrn   c                ó€   — | j                  «       sy| j                  |«      | j                  j                  |d«      k7  S )z»True if buf_name was produced on a different stream than node.

        Resolves mutation renames so that mutated buffers inherit the
        stream of their original definition.
        Fr   )rÂ  rÅ  r¡  rÈ  )rk   r  r�   s      rl   rB  z!Scheduler.has_cross_stream_hazardE  s<   € ð ×+Ñ+Ô-ØØ×"Ñ" 8Ó,°×0CÑ0C×0GÑ0GÈÈaÓ0PÑPÐPrn   c                ó6   — t         j                  j                  S rw   ©rZ   rœ   Úcurrent_devicerj   s    rl   rÉ  zScheduler.current_deviceO  s   € ä�w‰w×%Ñ%Ð%rn   c                ó.   — |t         j                  _        y rw   rÈ  r  s     rl   rÉ  zScheduler.current_deviceS  s   € à!'Œ�‰Õrn   c                ó€   — t         j                  j                  dd«      dk(  rddlm}  || j
                  d¬«       yy)z,Generate an image of the graph for debuggingÚINDUCTOR_WRITE_SCHEDULER_GRAPHNÚ1r   )Údraw_buffersT)Úprint_graph)ÚosÚenvironrÈ  r  rÎ  rÒ  )rk   rÎ  s     rl   r¦  zScheduler.debug_draw_graphW  s1   € ä�:‰:�>‰>Ð:¸DÓAÀSÒHÝ+á˜Ÿ™°Ö6ð Irn   c                óº   — t         j                  t        j                  «      r8t         j	                  d|«       | j
                  D ]  }|j                  «        Œ y y )Nz%s:)r“  ÚisEnabledForÚloggingÚINFOr©  rÒ  rª  )rk   Úlabelr�   s      rl   Údebug_print_nodeszScheduler.debug_print_nodes^  sF   € Ü×ÑœGŸL™LÔ)Ü�H‰H�U˜EÔ"ØŸ
™
ò #�Ø× Ñ Õ"ñ#ð *rn   c                ó6  — |j                  «       €J d«       ‚|j                  «       rt        | |«      S t        |t        j
                  t        j                  f«      rt        | |«      S t        |t        j                  «      rt        | |«      S t        |«      ‚)Nz2All nodes passed to scheduling must have an origin)rW  Úis_no_oprA  rŒ   r+   r   ra  r�   r  r�  r+  r%  s     rl   rr  zScheduler.create_scheduler_noded  s‡   € Ø×ÑÓ!Ð-ð 	
Ø@ó	
Ð-ð �=‰=Œ?Ü)¨$°Ó5Ð5Ü˜œr×0Ñ0´"×2CÑ2CÐDÔEÜ   tÓ,Ð,Ü˜œbŸo™oÔ.Ü,¨T°4Ó8Ð8ä% dÓ+Ð+rn   c                ó–  — t        «       }g }| j                  j                  «       }t        j                  j
                  j                  «       D ]«  }|D �cg c]%  }||v rt        | j                  |   t        «      s|‘Œ' }}|sŒ6|j                  |«       |D �cg c]  }| j                  |   ‘Œ }}t        j                  dkD  }t        | |d|¬«      }|j                  |«       |D ]  }|| j                  |<   Œ Œ­ | j                  D �	cg c]  }	|	j!                  «       |vsŒ|	‘Œ c}	t#        |«      z   | _        y c c}w c c}w c c}	w )Nr   F©r„  r‡  )r   ró  r   rZ   rœ   Úlistsr¹   rŒ   ru  rA  r  r(   Úcombo_kernels_autotunero  rÀ   rÒ  r  rÿ   )
rk   Úremoved_node_namesÚfe_nodesÚkept_node_namesÚnamesr²   r·   r‡  Úfe_noder�   s
             rl   r�  zScheduler.create_foreach_nodesq  sN  € Ü.8«lÐØˆØ×1Ñ1×6Ñ6Ó8ˆä—W‘W—]‘]×)Ñ)Ó+ò 	8ˆEð "öàØ˜?Ñ*Ü" 4×#4Ñ#4°TÑ#:Ô<RÔSò ðˆEð ñ àà×%Ñ% eÔ,Ø:?Ö@°$�d×'Ñ'¨Ó-Ð@ˆFÐ@ä$×;Ñ;¸aÑ?ˆOÜ0ØØØ*/Ø /ô	ˆGð �O‰O˜GÔ$àò 8�Ø07�×'Ñ'¨Ò-ñ8ð1	8ð8 "ŸZ™Zö
Ø¨4¯=©=«?ÐBTÒ+TŠDò
ä�‹Nñˆ�
ùò5ùò Aùò
s   Á*D<ÂEÄEÄ#Ec                ó*  ‡ ‡'‡(‡)—  G ˆ'fd„dt         t           «      Š't        j                  ‰'«      Š(‰ j                  D ]Ï  }|j                  «       D ]º  }|j                  «       }t        |j                  j                  t        j                  «      rt        |j                  «       «      dkD  rŒ^|j                  «       D ]J  }|‰(v r/|‰(v r+‰(|   }‰(|   }||z   }‰(D ]  }‰(|   |u s‰(|   |u sŒ|‰(|<   Œ Œ6|‰(v r	‰(|   ‰(|<   ŒC‰(|   ‰(|<   ŒL Œ¼ ŒÑ dˆ)ˆ fd„Š)	 	 d	 	 	 	 	 	 	 	 	 dˆ(ˆ)fd„}	i }
t        j                  j                   j#                  «       D ]£  }t        |t$        j&                  «      r|j(                  D ]  }d|
|<   Œ	 Œ4t        |t        j*                  «      sŒO|j-                  «       D �cg c]  }t        |t$        j&                  «      sŒ|‘Œ! }}|D ]  }|j(                  D ]  }d|
|<   Œ	 Œ Œ¥ d}‰ j                  D ]s  }|j                  €J ‚t/        |j                  j1                  «       d„ ¬	«      }|D ]8  }t        |t$        j2                  «      sJ ‚d
}||
vsŒ&|j                  «       |
|<   Œ: Œu ‰ j                  D �]@  }t4        j7                  d|j                  «       |rŸ|j                  €J ‚t/        |j                  j9                  d
¬«      d„ ¬	«      }|D ]d  }||
v sJ |› d|
› �«       ‚|
|   x}€Œ‰ j:                  |   j                  «       D ]*  }|j=                  t?        |j                  «       «      «       Œ, Œf t        |j@                  jB                  «      dk(  rGtE        tG        |j@                  jB                  «      «      x}rt        |tH        «      r|jJ                  }nd}|j                  «       D �]7  }t        |jM                  «       «      dk  sJ ‚|jM                  «       D �]  } ‰)|«      } |	||«       |j=                  t?        ||¬«      «       ‰(|   jN                  D ]À  }|j                  «       |j                  «       k(  rŒ%t        |j                  tP        «      sJ ‚|j                  j                  «       D ]c  }|j                  «       } ‰)|«      }||j                  «       v }|j=                  tS        ||j                  «       | ¬«      «        |	||d
¬«       Œe ŒÂ �Œ �Œ: t        j                  jT                  |j                  «          D ]8  } |	||d
¬«       |j=                  tS        ||j                  «       d
¬«      «       Œ: t        j                  jV                  |j                  «          D ]'  } |	||d¬«       |j=                  t?        |«      «       Œ) |j@                  jX                  D ]6  }t        |tR        «      rŒ |	|jZ                  ||j]                  |«      «       Œ8 |j_                  ‰ j`                  «       |j                  «       D ]Ž  }|jM                  «       D ]y  }|j                  «       ‰ j`                   ‰)|«      <   |j                  «       ‰ j`                  |<   ‰ jb                  je                  ||«      ‰ jb                  |j                  «       <   Œ{ Œ� �ŒC t        j                  jg                  «       D ]3  }t4        j7                  d|«        |	|ti        t?        |«      «      «       Œ5 |r°t        j                  jj                  D ]“  }|j9                  d
¬«      D ]|  }||
v sJ |› d|
jm                  «       › �«       ‚|
|   x}sŒ)‰ j:                  |   jo                  «       D ]4  }t4        j7                  d||«        |	|ti        t?        |«      «      «       Œ6 Œ~ Œ• ‰ j`                  D ]›  }|t        j                  j                   v rE |	|ti        t?        |«      «      «       t        j                  jp                  js                  |«       Œd|t        j                  jt                  v sŒ� |	|ti        t?        |«      «      «       Œ� tw        t        j                  j                   jm                  «       «      D � �ci c]  \  } }|| “Œ
 }!} }t        j                  jp                  D �cg c]  }|!|   ‘Œ	 c}t        j                  _<        ‰ j                  D ]C  }|j                  «       D ].  }|j{                  ‰(|j                  «          jN                  «       Œ0 ŒE ‰ j|                  D ]-  }‰ j|                  |   j{                  ‰(|   jN                  «       Œ/ t        «       }"|"j�                  d«       ‰(jO                  «       D ]]  \  }}#|"jƒ                  «       5  |#jN                  D �$cg c]  }$|$j                  «       ‘Œ }%}$|"j�                  d|› d|%› d�«       ddd«       Œ_ |"j�                  d«       |"j…                  «       j‡                  «       }&tˆ        j7                  d«       tˆ        j7                  d|&«       yc c}w c c}} w c c}w c c}$w # 1 sw Y   ŒÙxY w)zi
        Create dependency edges between nodes, handling aliasing and
        mutation properly.
        c                  ó>   •— e Zd ZdZ	 	 d	 	 	 	 	 dd„Zdd„Zd	ˆ fd„Zy)
ú1Scheduler.compute_dependencies.<locals>.DedupListan  
            This data structure behaves like a list except it makes sure the
            elements remain unique.
            Normally one could use a OrderedSet/dict for this purpose however
            the list in question gets elements appended as it is being
            iterated over which means that we need to keep the list
            semantics.
            Nc                ó@   — |xs g | _         |xs
 t        «       | _        y rw   )r�  r   Ú
membership)rk   r�  rç  s      rl   rx  z:Scheduler.compute_dependencies.<locals>.DedupList.__init__¥  s   € ð
 #š[ b�”
Ø",Ò"<´
³�•rn   c                óŽ   — || j                   v ry | j                  j                  |«       | j                   j                  |«       y rw   )rç  r�  rÀ   rÜ  )rk   Ú	node_users     rl   rÀ   z8Scheduler.compute_dependencies.<locals>.DedupList.append­  s5   € Ø §¡Ñ/ØØ—
‘
×!Ñ! )Ô,Ø—‘×#Ñ# IÕ.rn   c                óà   •— t        j                  | j                  |j                  «      }| j                  |j                  D �cg c]  }|| j                  vsŒ|‘Œ c}z   } ‰||«      S c c}w rw   )r   rá  rç  r�  )rk   rN  Únew_membershiprJ  Ú	new_itemsÚ	DedupLists        €rl   Ú__add__z9Scheduler.compute_dependencies.<locals>.DedupList.__add__³  sc   ø€ Ü!+×!1Ñ!1°$·/±/À5×CSÑCSÓ!T�Ø ŸJ™JØ$Ÿ{™{ö*Ø¨a°t·±Ò.F’Aò*ñ �	ñ ! ¨NÓ;Ð;ùò*s   ÁA+ÁA+rÕ  )r�  zlist[_T] | Nonerç  zOrderedSet[_T] | Noner†   rU  )ré  ra   r†   rU  )rN  úDedupList[_T]r†   rï  )ry   rz   r{   r  rx  rÀ   rî  )rí  s   €rl   rí  rå  ›  s;   ø„ ñð *.Ø48ð=à&ð=ð 2ð=ð ó	=ó/ö<rn   rí  r   c                óN   •— | ‰j                   v r ‰‰j                   |    «      S | S rw   )rg  )r  r¸  rk   s    €€rl   r¸  z.Scheduler.compute_dependencies.<locals>.renameà  s,   ø€ Ø�D×)Ñ)Ñ)Ù˜d×3Ñ3°AÑ6Ó7Ð7ØˆHrn   Fc                óP   •— ‰ ‰| «         j                  t        |||«      «       y rw   )rÀ   r]  )Úused_by_namer5  r(  r  Úname_to_usersr¸  s       €€rl   Úadd_userz0Scheduler.compute_dependencies.<locals>.add_userå  s)   ø€ ð ™& Ó.Ñ/×6Ñ6Ü˜ K°Ó9õrn   Nc                ó   — | j                   S rw   r³  rï  s    rl   rv  z0Scheduler.compute_dependencies.<locals>.<lambda>  s
   € ÀAÇFÁF€ rn   r$  Tzscheduling %s)Úunbacked_onlyc                ó   — | j                   S rw   r³  rï  s    rl   rv  z0Scheduler.compute_dependencies.<locals>.<lambda>  s
   €  !§&¡&€ rn   z not in )rÍ  ©Úmutating_bufrÚ  )r  )rÚ  zscheduling output %sz+scheduling output %s for unbacked symint %srS  ú'z': r$  rT  zBUFFER USER LIST
z===== AFTER SCHEDULING =====
%s)r  r  r†   r  )FF)
rò  r  r5  r  r(  rv   r  rv   r†   rU  )Er	   ra   r•  r   rÒ  rz  r  rŒ   r�   r'  r+   r?   r©   r(  rZ   rœ   r–  r¹   rä   rŽ  r"   Ú	TensorBoxr¢  r)  Úget_unbacked_symbol_defsÚSymbolr“  r  r6  ru  r¾  r5   r°   r�  rÐ  rÈ  r4   rÍ  r*  r�  r_   r6   Úadditional_buffer_depsÚadditional_star_depsr±   r²   r(  rº  rg  rÊ  rÈ  r•  rG  Úgraph_outputsr   r  Úmutated_inputsrÜ  rp  rÑ  Úmutated_input_idxsrM  r?  rR   rŽ  r+  r,  r•  Úcompute_dependencies_log)*rk   r�   Úbuf1Ú	buf1_nameÚ	buf2_nameÚlist1Úlist2Úcombinedr%  rô  Úunbacked_symbol_to_origin_nodeÚvalÚfsrt  Úsym_sizeÚhas_non_input_unbacked_defsÚunbacked_symbol_defsÚunbacked_symbol_usesr~  rº   r¼   Ú	node_modeÚalt_namer.  Úout_bufÚ
other_nameÚis_aliasÚadd_deprH  r  rÁ   r²   r³   Ú	inp_namesÚlogbufr©  r’  r  r  rí  ró  r¸  s*   `                                      @@@rl   r]   zScheduler.compute_dependencies•  sB
  û€ ö	<œ¤™ô 	<ô@ @K×?VÑ?VØó@
ˆð —J‘Jò 	LˆDØ×(Ñ(Ó*ò L�Ø ŸM™M›O�	ô ˜tŸy™y×/Ñ/´·±Ô?Ü˜D×,Ñ,Ó.Ó/°!Ò3àØ!%×!1Ñ!1Ó!3ò L�IØ  MÑ1°iÀ=Ñ6Pà -¨iÑ 8˜Ø -¨iÑ 8˜Ø#(¨5¡=˜Ø#0ò >˜Cà -¨cÑ 2°eÑ ;Ø#0°Ñ#5¸Ò#>à5= ¨cÒ 2ñ>ð # mÑ3Ø3@ÀÑ3K˜ iÒ0à3@ÀÑ3K˜ iÒ0ñLñLð	Lö<	ð !&Ø!ð		àð		ð 6ð		ð ð			ð
 ð		ð ö		ð JLÐ&ô
 —7‘7×'Ñ'×.Ñ.Ó0ò 
	BˆCÜ˜#œuŸz™zÔ*Ø×*Ñ*ò >�BØ9=Ð2°2Ò6ñ>ä˜C¤§¡Õ.ð (+§|¡|£~ÖS !¼ÀAÄuÇzÁzÕ9RšAÐS�ÐSØ!ò B�AØŸn™nò B˜Ø=AÐ6°rÒ:ñBñBð
	Bð ',Ð#Ø—J‘Jò 	HˆDØ—9‘9Ð(Ð(Ð(ô $*Ø—	‘	×2Ñ2Ó4Ñ:Jô$Ð ð *ò H�Ü! !¤U§\¡\Ô2Ð2Ð2ð /3Ð+ØÐ:Ò:Ø8<¿¹»Ð2°1Ò5ñHð	Hð  —J‘Jó V	ˆDÜ�I‰I�o t§y¡yÔ1á*Ø—y‘yÐ,Ð,Ð,ä'-Ø—I‘I×2Ñ2ÀÐ2ÓFÙ(ô(Ð$ð
 .ò G�AØÐ >Ñ>ð Ø˜#˜XÐ&DÐ%EÐFóÐ>ð <¸AÑ>Ð>˜ÑKØ#'×#4Ñ#4°QÑ#7×#CÑ#CÓ#Eò G˜CØ ×-Ñ-¬g°c·l±l³nÓ.EÕFñGðGô �D×$Ñ$×+Ñ+Ó,°Ò1Ü ¤ d×&6Ñ&6×&=Ñ&=Ó!>Ó?Ð?�SÐ?Ü˜s¤IÔ.àŸH™H‘	à �	ð ×'Ñ'Ó)ó E�ä˜3×,Ñ,Ó.Ó/°1Ò4Ð4Ð4Ø #× 1Ñ 1Ó 3ó E�HÙ% hÓ/�Há˜X tÔ,Ø×%Ñ%¤g¨h¸YÔ&GÔHØ -¨hÑ 7× =Ñ =ò E˜ØŸ=™=›?¨d¯m©m«oÒ=Ø$ä)¨$¯)©)Ô5FÔGÐGÐGØ'+§y¡y×'<Ñ'<Ó'>ò E˜GØ)0×)9Ñ)9Ó);˜Já)/°
Ó);˜Jð (0°7×3FÑ3FÓ3HÐ'H˜HØ ×-Ñ-Ü 'Ø$.Ø14·±³Ø08¨Lô!"ôñ % Z°¸tÖDñ%EòEòEðEôB Ÿ7™7×9Ñ9¸$¿-¹-»/ÑJò S�Ù˜ $°Õ5ð ×!Ñ!¤'¨'°4·=±=³?ÈDÔ"QÕRð	Sô Ÿ7™7×7Ñ7¸¿¹»ÑHò 4�Ù˜ $°Õ6Ø×!Ñ!¤'¨'Ó"2Õ3ð4ð
 ×(Ñ(×.Ñ.ò F�Ü! $¬Õ0Ù˜TŸY™Y¨¨d×.>Ñ.>¸tÓ.DÕEðFð ×%Ñ% d×&;Ñ&;Ô<ð ×'Ñ'Ó)ò �Ø #× 1Ñ 1Ó 3ò �HØ>A¿l¹l»n�D×)Ñ)©&°Ó*:Ñ;Ø69·l±l³n�D×)Ñ)¨(Ñ3à×/Ñ/×3Ñ3°H¸hÓGð ×+Ñ+¨C¯L©L«NÒ;ñòðaV	ôr Ÿ™×0Ñ0Ó2ò 	>ˆHÜ�I‰IÐ,¨hÔ7Ù�Xœz¬'°(Ó*;Ó<Õ=ð	>ñ
 'Ü—w‘w×,Ñ,ò N�Ø×1Ñ1ÀÐ1ÓEò N�AØÐ >Ñ>ð Ø˜#˜XÐ&D×&IÑ&IÓ&KÐ%LÐMóÐ>ð ;¸1Ñ=Ð=�qÑ=Ø(,×(9Ñ(9¸!Ñ(<×(MÑ(MÓ(Oò N˜HÜŸI™IØ MØ (Ø !ôñ
 % X¬z¼'À(Ó:KÓ/LÕMñNñNðNð ×)Ñ)ò 	:ˆDØ”q—w‘w×+Ñ+Ñ+Ù˜œz¬'°$«-Ó8Ô9Ü—‘×&Ñ&×*Ñ*¨4Õ0ØœŸ™×*Ñ*Ò*á˜œz¬'°$«-Ó8Õ9ð	:ô ,5´Q·W±W×5IÑ5I×5NÑ5NÓ5PÓ+Q÷
Ù'˜E 4ˆD�%‰Kð
ˆ	ñ 
ô )*¯©×(>Ñ(>ö&
Ø $ˆI�d‹Oò&
Œ�‰Ô"ð
 —J‘Jò 	CˆDØ×'Ñ'Ó)ò C�Ø—‘˜m¨C¯L©L«NÑ;×AÑAÕBñCð	Cð ×/Ñ/ò 	SˆDØ×'Ñ'¨Ñ-×7Ñ7¸ÀdÑ8K×8QÑ8QÕRð	Sô  Ó!ˆØ�‰�cÔØ'×-Ñ-Ó/ò 	4‰JˆC�Ø—‘“ñ 4Ø/4¯{©{Ö;¨!˜Ÿ™�Ð;�Ð;Ø—‘  #  c¨%¨°Ð2Ô3÷4ð 4ð	4ð 	�‰�cÔØ× Ñ Ó"×)Ñ)Ó+ˆÜ ×&Ñ&Ð';Ô<Ü ×&Ñ&Ð'IÈ3ÕOùòK TùóX
ùò&
ùò" <÷4ð 4ús6   Æ5i4Çi4â2i9ãi?çj	çjç5j	êj	ê	j	c           
     ó  ‡ ‡— ddl m}m}m}m} t        t        j                  j                  j                  «       «      } |‰ j                  |«      }t        j                  j                  j                  s |‰ j                  ‰ j                  «       t        t        j                  j!                  «       «      } |‰ j                  ||«      \  }}	}	t#        t%        ‰ j                  «      «      D �	cg c]  }	g g f‘Œ c}	Š|D ]}  }
|
j&                  dk(  r|
j(                  dk(  rŒ"|
j*                  j-                  «       }‰|
j.                     d   j1                  |«       ‰|
j2                     d   j1                  |«       Œ ddlm}  |«        	 	 	 	 	 	 dˆ ˆfd„}g }t9        ‰ j                  «      D ]H  \  }}|j1                  |«       |j1                   |||t%        ‰ j                  «      dz
  k(  ¬«      «       ŒJ |‰ _
        y c c}	w )Nr   )rZ  Úcompute_memory_timelineÚFreeableInputBufferÚget_freeable_input_bufr   )Úregister_check_mem_opc                óX  •— ‰|    d   }‰|    d   }|||g}t        j                  t        t        j                  d«      ¬«      t        j
                  j                  j                  j                  g |d„ ¬«      }d‰j                  |    j                  «       › �|_        t        ‰|«      S )Nr   r   r  r·  c                ó$   — | |d   |d   |d   dœfS )Nr   r   r¨   )ÚaliveÚdeadÚis_final_stepr~   )Útensor_argsrõ  s     rl   rv  zWScheduler.insert_memory_check_nodes.<locals>.construct_mem_check_node.<locals>.<lambda>Û  s*   € Øà!.¨qÑ!1Ø -¨aÑ 0Ø)6°qÑ)9ñðC€ rn   )r'  r=  r#  Únontensor_argsÚunflatten_argsÚ
mem_check_)r+   ÚMemoryCheckKernelr?   rŸ  r  rü  Ú_inductor_debugÚcheck_memory_stepÚdefaultrÒ  r  rô  r�  )Ústep_idxr"  Úexpected_newly_aliveÚexpected_newly_deadr$  r�   rk   Ústep_allocs_deallocss         €€rl   Úconstruct_mem_check_nodezEScheduler.insert_memory_check_nodes.<locals>.construct_mem_check_nodeÎ  s¨   ø€ ð $8¸Ñ#AÀ!Ñ#DÐ Ø"6°xÑ"@ÀÑ"CÐà2Ð4GÈÐWˆNä×'Ñ'Ü!¬¯©°eÓ)<Ô=Ü—y‘y×0Ñ0×BÑB×JÑJØØ-ñ ôˆDð %/¨t¯z©z¸(Ñ/C×/LÑ/LÓ/NÐ.OÐ"PˆDÔÜ,¨T°4Ó8Ð8rn   )r"  )r+  r  r"  rv   r†   r�  )r”  rZ  r  r  r  r   rZ   rœ   r–  r   rÒ  rŸ  r   r(   rY  r@  r•  r„  r©   Ú
size_allocÚ	size_freer]  r  Ú
start_steprÀ   Úend_stepÚ#torch._inductor.runtime.debug_utilsr  rÑ  )rk   rZ  r  r  r  r–  Úname_to_freeable_input_bufr   Úbuf_info_listr  Úbuf_infor  r  r/  Ú	new_nodesrÏ  r�   r.  s   `                @rl   r¤  z#Scheduler.insert_memory_check_nodes¤  sá  ù€ ÷	
ó 	
ô )3´1·7±7×3GÑ3G×3LÑ3LÓ3NÓ(Oˆá" 4§:¡:¨|Ó<ð 	#ô �‰×%Ñ%×=Ò=Ù=Ø—
‘
˜D×,Ñ,ôô *4´A·G±G×4LÑ4LÓ4NÓ)OˆÙ5Ø�J‰JØ&Øó
Ñˆ�q˜!ô $¤C¨¯
©
£OÓ4öC
ØˆR�ŠHòC
Ðð &ò 	HˆHà×"Ñ" aÒ'¨H×,>Ñ,>À!Ò,CØà—‘×/Ñ/Ó1ˆHà  ×!4Ñ!4Ñ5°aÑ8×?Ñ?ÀÔIØ  ×!2Ñ!2Ñ3°AÑ6×=Ñ=¸hÕGð	Hõ 	NáÔð	9Øð	9Ø*.ð	9à&ö	9ð2 ˆ	ä  §¡Ó,ò 	‰GˆAˆtØ×Ñ˜TÔ"Ø×ÑÙ(¨¸1ÄÀDÇJÁJÃÐRSÑ@SÑ;SÔUõð	ð ˆ�
ùòeC
s   Ã3Hc                ó|  ‡	— t         j                  syg }t        | j                  «      D �]Ð  }dd„Š	d}|j	                  «       D ]€  }t        ˆ	fd„|j                  D «       «      }|r\t        j                  d|j                  «       «       t        j                  j                  j                  |j                  «       «       Œd}Œ‚ |j                  «        xr | }|s|j                  |«       ŒÇt        j                  d|j                  «       «       t        j                  j                   j                  |j                  «       «       |j"                  j$                  D ]–  }|j&                  | j(                  v sŒ| j(                  |j&                     j                  }|D �cg c]0  }|j*                  j                  «       |j                  «       k7  sŒ/|‘Œ2 c}| j(                  |j&                     _        Œ˜ �ŒÓ t-        t        |«      «      | _        | j                  D ]  }|j/                  «        Œ yc c}w )	z0
        Remove any nodes without users
        Nc                ór   — | j                   xs* | j                  «       t        j                  j                  v S rw   )r  r  rZ   rœ   rç  )r.  s    rl   Úcan_eliminate_userz;Scheduler.dead_node_elimination.<locals>.can_eliminate_userþ  s&   € Ø—|‘|ÒT t§}¡}£¼!¿'¹'×:TÑ:TÐ'TÐTrn   Fc              3  ó.   •K  — | ]  } ‰|«      –— Œ y ­wrw   r~   )r’   Úur;  s     €rl   r”   z2Scheduler.dead_node_elimination.<locals>.<genexpr>  s   øè ø€ Ò#M¸aÑ$6°q×$9Ñ#Mùó   ƒzremoved dead buffer: %sTzremoved dead operation: %s)r.  r]  r†   rv   )r(   Úuse_dcer«   rÒ  rz  r•   r  r“  r  r  rZ   rœ   r?  rÜ  r*  rÀ   rç  r°   r±   r²   r@  r�   rÿ   rð  )
rk   Úupdated_nodesr�   Úactive_buffersrº   Úcan_eliminaterH  r  r=  r;  s
            @rl   r|  zScheduler.dead_node_eliminationñ  sÂ  ø€ ô �~Š~Øð
 ˆÜ˜TŸZ™ZÓ(ó 	ˆDóUð #ˆNØ×'Ñ'Ó)ò *�Ü #Ó#MÀ3Ç9Á9Ô#MÓ M�Ù Ü—I‘IÐ7¸¿¹»ÔHÜ—G‘G×+Ñ+×/Ñ/°·±³Õ?à%)‘Nð*ð !%× 5Ñ 5Ó 7Ð7ÒNÀÐ<NˆMá Ø×$Ñ$ TÕ*ô —	‘	Ð6¸¿¹»ÔHÜ—‘×*Ñ*×.Ñ.¨t¯}©}«Ô?Ø ×,Ñ,×2Ñ2ò �DØ—y‘y D×$4Ñ$4Ò4Ø $× 0Ñ 0°·±Ñ ;× AÑ A˜à',ö=Ø"#°·±·±Ó0AÀTÇ]Á]Ã_Ó0TšAò=˜×(Ñ(¨¯©Ñ3Õ9òð-	ô8 œ( =Ó1Ó2ˆŒ
ð —J‘Jò 	#ˆDØ× Ñ Õ"ñ	#ùò=s   Æ%0H9ÇH9c                ó
   — |duS )z:Check if store mode requires cross-thread synchronization.Nr~   )rk   rÍ  s     rl   Úmode_requires_synchronizationz'Scheduler.mode_requires_synchronization  s   € à˜4ÐÐrn   c                óº   ‡‡‡‡— t        t           «       Št        «       Šg Šdˆˆˆˆfd„Š|D ]  }|j                  «       D ]  }|‰|<   Œ	 Œ |D ]
  } ‰|«       Œ ‰S )z?
        Ensure nodes is in topologically sorted order
        c                óÖ   •— | ‰vrd‰j                  | «       t        | j                  d„ ¬«      D ]&  }|j                  ‰vrŒ ‰‰|j                     «       Œ( ‰j	                  | «       y y )Nc                ó   — | j                   S rw   r³  )Úds    rl   rv  zDScheduler.topological_sort_schedule.<locals>.visit.<locals>.<lambda>/  s
   € ÀaÇfÁf€ rn   r$  )rÜ  r)  rq  r²   rÀ   )r  r¼   ru  r-  ÚseenÚvisits     €€€€rl   rJ  z2Scheduler.topological_sort_schedule.<locals>.visit,  se   ø€ Ø˜‰}Ø—‘˜”Ü! !×"6Ñ"6Ñ<LÔMò 2�Cà—x‘x |Ñ3Ø Ù˜, s§x¡xÑ0Õ1ð	2ð
 —‘˜aÕ ð rn   )r  r_   r†   rU  )r   r_   r°  r  )rk   rÒ  r�   r²   ru  r-  rI  rJ  s       @@@@rl   r{  z#Scheduler.topological_sort_schedule"  sy   û€ ô Ô+Ñ,Ó.ˆÜ59³VˆØ*,ˆ÷	!ð 	!ð ò 	*ˆDØ×-Ñ-Ó/ò *�Ø%)�˜TÒ"ñ*ð	*ð ò 	ˆDÙ�$�Kð	àˆrn   c                ó˜  — | j                   D �cg c])  }t        |j                  t        j                  «      sŒ(|‘Œ+ }}t        dt        |«      «      D ]o  }t        t        ||   j                  «       «      «      }t        t        ||dz
     j                  «       «      «      }||   j                  t        ||d¬«      «       Œq y c c}w )Nr   Trø  )rÒ  rŒ   r�   r+   ÚConditionalr„  r©   rÐ  rÈ  r  r¾  r6   )rk   r  Úconditional_nodesrÏ  rù  Úprev_bufs         rl   r“  z'Scheduler._enforce_conditional_ordering=  s´   € à—z‘zö
Ø¤Z°·±¼¿¹Õ%GŠAð
Ðð 
ô �qœ#Ð/Ó0Ó1ò 	ˆAÜ¤Ð%6°qÑ%9×%JÑ%JÓ%LÓ MÓNˆLÜœDÐ!2°1°q±5Ñ!9×!JÑ!JÓ!LÓMÓNˆHØ˜aÑ ×-Ñ-Ü˜¨|ÀTÔJõñ	ùò
s
   �)C¹Cc                ó<  ‡ — t        «       }t        |t        t        t        t
        t        f«      r-|j                  D ]  }|j                  |j                  «       Œ nt        dt        |«      › d�«      ‚ˆ fd„|D «       }t        t        ˆ fd„|D «       «      «      S )Nz+get_unmet_dep_nodes is not implemented for ú.c              3  óX   •K  — | ]!  }‰j                   |   j                  «       –— Œ# y ­wrw   )r@  r  rã  s     €rl   r”   z1Scheduler._get_unmet_dep_nodes.<locals>.<genexpr>Z  s%   øè ø€ ÒXÀc˜×)Ñ)¨#Ñ.×?Ñ?×AÑXùs   ƒ'*c              3  ó<   •K  — | ]  }‰j                   |   –— Œ y ­wrw   ©ró  )r’   r  rk   s     €rl   r”   z1Scheduler._get_unmet_dep_nodes.<locals>.<genexpr>[  s   øè ø€ ÒQ¸a˜t×6Ñ6°qÕ9ÑQùó   ƒ)r   rŒ   r�   r�  rA  rµ   rì  rq  rÜ  r²   ÚRuntimeErrorr¶   rÿ   )rk   rø  Ú
unmet_depsr¼   Úunmet_dep_opss   `    rl   Ú_get_unmet_dep_nodeszScheduler._get_unmet_dep_nodesH  s’   ø€ Ü&0£lˆ
ÜØäÜ)Ü&Ü"Ü$ðô	
ð ×/Ñ/ò )�Ø—‘˜sŸx™xÕ(ñ)ô Ø=¼dÀ5»k¸]È!ÐLóð ó YÈZÔXˆÜ”JÓQÀ=ÔQÓQÓRÐRrn   c                óz  — g }t         j                  | j                  d«      }i }| j                  D ]P  }| j                  |«      }t	        |«      ||<   |D ]*  }|j                  |g «      }|j                  |«       |||<   Œ, ŒR |j                  «       D ��	cg c]  \  }}	|	dk(  sŒ|‘Œ }
}}	|
rx|j                  |
«       |
D ]7  }|j                  |g «      D ]  }||xx   dz  cc<   Œ |j                  |«       Œ9 |j                  «       D ��	cg c]  \  }}	|	dk(  sŒ|‘Œ }
}}	|
rŒx|rJ d«       ‚|S c c}	}w c c}	}w )zU
        Sort nodes by their topological order, return a list of node lists.
        r   r   zTopological sort failed!)	r°  ÚfromkeysrÒ  rX  r©   rÈ  rÀ   r�  rÛ  )rk   rï  rÒ  Úchildrenr�   rß  r¼   Úcr  r’  Úzero_deg_nodesr.  s               rl   r   z!Scheduler._topological_sort_nodes]  sF  € ð ˆÜ—‘˜dŸj™j¨!Ó,ˆØ#%ˆØ—J‘Jò 	"ˆDØ×,Ñ,¨TÓ2ˆDÜ˜d›)ˆE�$‰KØò "�Ø—L‘L  bÓ)�Ø—‘˜”Ø !�˜’ñ"ð	"ð ).¯©«×@¡  1¸¸a»š!Ð@ˆÑ@ÙØ�L‰L˜Ô(Ø#ò �Ø$ŸL™L¨¨BÓ/ò %�DØ˜$“K 1Ñ$”Kð%à—	‘	˜!•ðð -2¯K©K«M×D¡D A q¸QÀ!»VšaÐDˆNÑDò ñ Ð4Ð4Ó4ˆyØˆùó Aùó Es   ÂD1Â%D1ÄD7ÄD7c                ój  — i }| j                   D ]w  }t        «       }|j                  D ]B  }| j                  |j                     j                  «       }|j                  |«       |||   z  }ŒD |||j                  «       <   ||_        Œy t        | j                   «      D ]  \  }}||_
        ||_        Œ y)z.
        Populate each node.ancestors
        N)rÒ  r   rq  r@  r²   r  rÜ  r  rß   rÑ  rc  rd  )rk   Úname_to_ancestorsr�   rß   r¼   Údep_node_namerï  s          rl   r}  zScheduler.compute_ancestorsw  s¸   € ð
 9;ÐØ—J‘Jò 	'ˆDÜ)3«ˆIØ×.Ñ.ò >�Ø $× 0Ñ 0°·±Ñ :× KÑ KÓ M�Ø—‘˜mÔ,ØÐ.¨}Ñ=Ñ=‘	ð>ð 2;Ð˜dŸm™m›oÑ.Ø&ˆD�Nð	'ô % T§Z¡ZÓ0ò 	#‰KˆE�4Ø"ˆDŒNØ"ˆD�Nñ	#rn   c                ó  — i }i }| j                   D ]å  }|j                  sd}d}nž|j                  D �cg c]/  }|| j                  |j                     j	                  «          dz   ‘Œ1 }}|j                  D �cg c]/  }|| j                  |j                     j	                  «          dz   ‘Œ1 }}t        |«      }t        |«      }|||j                  «       <   |||j                  «       <   ||_        ||_	        Œç yc c}w c c}w )zí
        Populate each node's min/max_input_distance with the depth from graph
        inputs, measured as dependency hops before fusion. Nodes whose
        dependencies are all satisfied by graph inputs/constants have depth 0.
        r   r   N)
rÒ  rq  r@  r²   r  r‰  r¶  r  ra  rb  )	rk   Úname_to_min_distanceÚname_to_max_distancer�   Úmin_distÚmax_distr¼   Údep_min_distsÚdep_max_distss	            rl   r~  z!Scheduler.compute_input_distancesŠ  s   € ð 02ÐØ/1ÐØ—J‘Jò 	/ˆDØ×*Ò*Ø�Ø‘ð
  $×6Ñ6ö!ð ð )¨×)9Ñ)9¸#¿(¹(Ñ)C×)TÑ)TÓ)VÑWØóð!�ð !ð  $×6Ñ6ö!ð ð )¨×)9Ñ)9¸#¿(¹(Ñ)C×)TÑ)TÓ)VÑWØóð!�ð !ô
 ˜}Ó-�Ü˜}Ó-�Ø4<Ð  §¡£Ñ1Ø4<Ð  §¡£Ñ1Ø&.ˆDÔ#Ø&.ˆDÕ#ñ)	/ùò
!ùò
!s   ³4C:Á74C?c                óH  — t         j                  sy | j                  D ]‚  }t        |t        t
        f«      r#|j                  «       st         j                  dk7  rŒ=|j                  «       D ]3  }t        |t        «      r|j                  «       rŒ$|j                  «        Œ5 Œ„ y )NÚhalide)r(   re  rÒ  rŒ   r�   rµ   rU   Úcpu_backendr–   r   r”  )rk   r�   rø  s      rl   r”  zScheduler.merge_loops¨  s†   € Ü×0Ò0Øà—J‘Jò 	$ˆDô ˜d¤]Ô4FÐ$GÔHØ—K‘K”M¤f×&8Ñ&8¸HÒ&DàØŸ™Ó)ò $�ä! %¬Ô7¸5×;LÑ;LÔ;NØà×!Ñ!Õ#ñ$ñ	$rn   c                óÚ  — t        ddd¬«      5  t        d«      D ]…  }t        |«      }t        j	                  d|dz   |«       | j                  |d¬«      }t        |«      }t        j	                  d	|dz   ||«       ||k(  s|dk(  sŒlt        j	                  d
|dz   «        n t        j                  st        j                  r| j                  |d¬«      }|cddd«       S # 1 sw Y   yxY w)zB
        Combine eligible nodes into FusedSchedulerNodes.
        zScheduler.fused_nodesTrU  r	  z/===== attempting fusion (%d/10): %d nodes =====r   F)Úis_reorder_roundz=completed fusion round (%d/10): fused %d nodes into %d nodes
z+===== fusion complete (%d iterations) =====N)	r   r„  r©   r  r  Úfuse_nodes_oncer(   re  Úloop_index_inversion_in_fusion)rk   rÒ  rÏ  Úold_lenÚnew_lens        rl   rÊ  zScheduler.fuse_nodesÃ  sø   € ô Ø#¸4ÐQUô
ñ 	ô ˜2“Yò �Ü˜e›*�Ü× Ñ ØEØ˜‘EØôð
 ×,Ñ,¨UÀUÐ,ÓK�Ü˜e›*�Ü× Ñ ØTØ˜‘EØØô	ð ˜gÒ%¨°A«Ü×$Ñ$ØEÀqÈ1Áuôñ ð'ô, ×1Ò1Ü×8Ò8à×,Ñ,¨UÀTÐ,ÓJ�Ø÷;	÷ 	ò 	ús   �A7C!ÂAC!Ã!C*c                óœ   — g }| j                   D ]4  }|j                  t        |t        «      r|j	                  «       n|g«       Œ6 || _         y)zA
        Unpack GroupedSchedulerNode into regular nodes.
        N)rÒ  rÞ  rŒ   rì  rË  )rk   r8  r�   s      rl   rœ  zScheduler.process_grouped_nodesæ  sJ   € ð .0ˆ	Ø—J‘Jò 	ˆDØ×ÑÜ!+¨DÔ2FÔ!G�—‘”ÈdÈVõð	ð ˆ�
rn   c                óâ   — t        |«      dkD  sJ ‚|d   j                  «       }|| _        | j                  |«      }t	        ddd¬«      5  |j                  |«      cddd«       S # 1 sw Y   yxY w)úƒ
        Benchmark fused list of nodes and return the execution time
        in milliseconds on randomly generated inputs.
        r   Úbenchmark_fused_nodesTÚcompile_time_autotune_time_us)rV  Údynamo_compile_column_usN)r©   rÞ   rÉ  rc  r   rt  )rk   rÒ  r  rQ  s       rl   rt  zScheduler.benchmark_fused_nodesñ  st   € ô �5‹z˜AŠ~Ðˆ~Ø�q‘×$Ñ$Ó&ˆØ$ˆÔØ×"Ñ" 6Ó*ˆÜØ#Ø"&Ø%Dô
ñ 	8ð
 ×0Ñ0°Ó7÷	8÷ 	8ò 	8ús   Á
A%Á%A.c                óâ   — t        |«      dkD  sJ ‚|d   j                  «       }|| _        | j                  |«      }t	        d«      5  |j                  |||¬«      cddd«       S # 1 sw Y   yxY w)rs  r   Úgenerate_kernel_code_from_nodes©Úhint_overrideN)r©   rÞ   rÉ  rc  r   rx  )rk   rÒ  Úbenchmark_kernelrz  r  rQ  s         rl   rx  z)Scheduler.generate_kernel_code_from_nodes  sw   € ô �5‹z˜AŠ~Ðˆ~Ø�q‘×$Ñ$Ó&ˆØ$ˆÔØ×"Ñ" 6Ó*ˆÜÐ;Ó<ñ 	Ø×:Ñ:ØÐ'°}ð ;ó ÷	÷ 	ò 	ús   ÁA%Á%A.c                ó–   — || _         | j                  |«      }t        d«      5  |j                  |«      cddd«       S # 1 sw Y   yxY w)rs  Úbenchmark_codegened_moduleN)rÉ  rc  r   r}  )rk   Úmoduler  rQ  s       rl   r}  z$Scheduler.benchmark_codegened_module  sH   € ð %ˆÔØ×"Ñ" 6Ó*ˆÜÐ6Ó7ñ 	>Ø×5Ñ5°fÓ=÷	>÷ 	>ò 	>ús	   ¤?¿Ac                ó  — t         j                  j                  }|syt        j	                  d||«       |j
                  D ]Ê  }|j                  «       }t        |dd«      r||vst        |t        j                  «      rŒ?|j                  }||   }t        |t        j                  «      r'|j                  |j                  «       |j                  }t        |t        j                  «      sŒ¬||k7  sŒ²t        j!                  d|||«        y y)z 
        Check if selecting a Triton template would cause layout conflicts.
        Returns True if there's a conflict and we should fall back to ATen.
        FzNode %s has constraints %sr'  NzOLayout conflict detected for %s: template expects %s but layout is frozen to %sT)rZ   rœ   Úbuffer_layout_constraintsr“  r  ró  r  r�  rŒ   r+   ÚReinterpretViewr'  ÚFlexibleLayoutÚ freeze_layout_with_exact_stridesr+  ÚFixedLayoutr”  )rk   Ú
multi_nodeÚconstraintsÚinpÚinp_namer'  Úexpected_layouts          rl   Ú!_has_layout_conflict_for_templatez+Scheduler._has_layout_conflict_for_template"  sì   € ô —g‘g×7Ñ7ˆÙØä�	‰	Ð.°
¸KÔHØ×$Ñ$ò 	ˆCà—|‘|“~ˆHô ˜C ¨4Ô0Ø ;Ñ.Ü˜c¤2×#5Ñ#5Ô6àà—Z‘ZˆFØ)¨(Ñ3ˆOÜ˜&¤"×"3Ñ"3Ô4ð ×4Ñ4°_×5KÑ5KÔLØŸ™�ä˜&¤"§.¡.Õ1°oÈÓ6Oä—‘ØeØØ#Øô	ñ ð7	ð: rn   c           
     ó¦  — t        | j                  «      D �]&  \  }}t        |t        «      sŒt        |j                  t
        j                  «      sŒ=|j                  }t        j                  j                  s|j                  «       \  }}n t        d„ |j                  «       D «       «      }t        |t        j                  j
                  j                  «      r�| j!                  |«      r||j                  «       D ]4  }t        |t        j                  j"                  j$                  «      sŒ2|} n t        t        j                  j"                  j$                  «      sJ d«       ‚t        |t        j                  j
                  j                  «      rÎt        j&                  r¡i }||d<   t        j&                  D ]k  }|j                  |¬«      }	|	j)                  «       D �
�ci c]  \  }
}t        |
t        «      r|
|“Œ }}
}t+        |j)                  «       d„ ¬«      d   }|||<   Œm |j                  j-                  |«       n|j                  j/                  |«       �ŒNt
        j0                  j3                  |j4                  «      5  |j7                  «       }ddd«       j8                  }t        |t
        j:                  «      sJ ‚|j8                  }t        |t
        j<                  «      sJ ‚|j>                  rtA        ||j>                  «       |jB                  |_!        | jE                  ||||«       �Œ) yc c}}
w # 1 sw Y   ŒªxY w)a€  
        Finalize a backing choice for MultiTemplateBuffers which did not already have a
        choice finalized through fusion. In the case of an extern choice, this will result
        in replacing the SchedulerNode.

        If a MultiTemplateBuffer did not have any fusion opportunities, finalizing a choice
        will force completion of compilation and benchmarking.
        c              3  ó|   K  — | ]4  }t        |t        j                  j                  j                  «      r|–— Œ6 y ­wrw   )rŒ   rŸ  r   rŽ  ÚExternKernelCaller)r’   Útimings     rl   r”   z<Scheduler.finalize_multi_template_buffers.<locals>.<genexpr>`  s6   è ø€ ò à &Ü)Ø &Ü %§¡× @Ñ @× SÑ Sô ô #ñùr÷  zZNo extern kernel detected to fallback to when layout constraints fail for Triton templatesNry  c                ó   — | d   S r[  r~   rï  s    rl   rv  z;Scheduler.finalize_multi_template_buffers.<locals>.<lambda>�  s   € ÈqÐQRÉt€ rn   r$  r   )#rÑ  rÒ  rŒ   r�   r�   r+   ÚMultiTemplateBufferr(   r¢  Ú%force_extern_kernel_in_multi_templateÚget_min_choicerÐ  Úchoice_timingsrŸ  r   r   rŠ  rŽ  r�  Úmulti_kernel_hintsr�  r‰  Úfinalize_as_triton_callersÚfinalize_as_triton_callerrª  Úcurrent_originsr_  Úoutput_noderÔ   Ú
StorageBoxÚOperationBufferÚorigin_noder:   r'  Ú_replace_node)rk   rÏ  r�   r…  Úmin_node_unfusedr  ÚchoiceÚcallersÚhintÚtimingsrÉ  r’  Útriton_timingsÚout_tensorboxÚout_storageÚ
out_buffers                   rl   r‹  z)Scheduler.finalize_multi_template_buffersM  sä  € ô ! §¡Ó,ó L	D‰GˆAˆtÜ˜$¤Õ.´:Ø—	‘	œ2×1Ñ1õ4ð "ŸY™Y�
Ü×*Ñ*×PÒPØ*4×*CÑ*CÓ*EÑ'Ð$¡aä'+ñà*4×*CÑ*CÓ*Eôó	(Ð$ô Ø$Ü—O‘O×&Ñ&×?Ñ?ôð
 ×=Ñ=¸jÔIà&0×&?Ñ&?Ó&Aò &˜FÜ)Ø &Ü %§¡× @Ñ @× SÑ Sõ ð 4:Ð 0Ù %ð&ô  *Ø"¤E§O¡O×$DÑ$D×$WÑ$Wô ð ð yóð  ô Ø$Ü—O‘O×&Ñ&×?Ñ?ôô
 ×0Ò0ØNP˜Ø(8˜ ™ô %+×$=Ñ$=ò 3˜DØ&0×&?Ñ&?ÈdÐ&?Ó&S˜Gð -4¯M©M«O÷.á$( A qÜ#-¨aÔ1IÔ#Jð !" 1¡ð.˜Nñ .ô
 &)¨×)=Ñ)=Ó)?Á^Ô%TÐUVÑ%W˜FØ,2˜G DšMð3ð Ÿ	™	×<Ñ<¸WÕEàŸ	™	×;Ñ;Ð<LÔMÙä—Y‘Y×.Ñ.¨z×/AÑ/AÓBñ CØ$4×$@Ñ$@Ó$B�M÷Cà+×0Ñ0�Ü! +¬r¯}©}Ô=Ð=Ð=Ø(×-Ñ-�
Ü! *¬b×.@Ñ.@ÔAÐAÐAà×)Ò)Ü& }°j×6LÑ6LÔMà$.×$5Ñ$5�
Ô!Ø×"Ñ" :¨z¸1¸dÖCñYL	Dùóh.÷Cð Cús   Ç%M
ÊMÍM	c                óŒ  ‡— t        ||«       | j                  |«      }|| j                  |<   || j                  |j	                  «       <   || j
                  |j	                  «       <   i Št        j                  |j                  j                  |j                  «      D ]:  }| j                  j                  |j                  d «      x}sŒ,|j                  ‰|<   Œ< dˆfd„} ||j                  «      |_
         ||j                  j                  «      |j                  _	        t        |j                  «       |j                  «       «      D ]3  \  }	}
|	| j                   |
j	                  «       <   |
j"                  |	_        Œ5 |j$                  |_        |j&                  |_        |j(                  |_        |j*                  |_        y )Nc                ó,   •— t        ˆfd„| D «       «      S )Nc              3  ó@   •K  — | ]  }|j                  ‰«      –— Œ y ­wrw   )r¸  )r’   r¼   rg  s     €rl   r”   z?Scheduler._replace_node.<locals>.rename_deps.<locals>.<genexpr>º  s   øè ø€ ÒK¸s˜cŸj™jÐ)9×:ÑKùr1  r   )rß  rg  s    €rl   Úrename_depsz,Scheduler._replace_node.<locals>.rename_deps¹  s   ø€ ÜÓKÀdÔKÓKÐKrn   )rß  rp  r†   rp  )rý  rr  rÒ  ru  r  ró  rÔ  rÕ  r°   r±   rq  rÊ  rÈ  r²   r±  rz  r@  r  rc  rd  rß   r`  )rk   r¥  r…  rÏ  r�   Únew_scheduler_noder¼   Ú	real_namer©  Únew_outÚold_outrg  s              @rl   rœ  zScheduler._replace_node¥  s�  ø€ ô 	" *¨jÔ9Ø!×7Ñ7¸
ÓCÐà*ˆ�
‰
�1‰Ø-?ˆ×Ñ˜$Ÿ-™-›/Ñ*Ø3Eˆ×Ñ §¡£Ñ0ð ÐÜ—?‘? 4×#3Ñ#3×#9Ñ#9¸4×;RÑ;RÓSò 	7ˆCØ ×3Ñ3×7Ñ7¸¿¹À$ÓGÐGˆyÑGØ.1¯h©hÐ  Ò+ð	7õ	Lñ 1<Ø×1Ñ1ó1
ÐÔ-ñ 0;Ø×*Ñ*×0Ñ0ó0
Ð×&Ñ&Ô,ô !$Ø×*Ñ*Ó,¨d×.>Ñ.>Ó.@ó!
ò 	*ÑˆG�Wð 4;ˆD×Ñ˜W×-Ñ-Ó/Ñ0Ø#ŸM™MˆG�Mð		*ð (,§~¡~ÐÔ$Ø'+§~¡~ÐÔ$Ø'+§~¡~ÐÔ$Ø(,¯©ÐÕ%rn   c                ó&   — t        d„ |D «       «      S )Nc              3  óö   K  — | ]q  }t        |j                  d «      xrU |j                  duxrE t        |j                  j                  d«      xr# |j                  j                  j                  dk(  –— Œs y­w)rÔ   NÚscatter_moderÎ  )r<  r�   rÔ   r°  r  s     rl   r”   z,Scheduler._any_atomic_add.<locals>.<genexpr>Ï  so   è ø€ ò 
ð
 ô	 �A—F‘F˜FÓ#ò 9Ø—‘˜dÐ"ò9ä˜Ÿ™Ÿ™ ^Ó4ò9ð —‘—‘×(Ñ(¨LÑ8ó9ñ
ùs   ‚A7A9)ræ   )rk   Ú	node_lists     rl   Ú_any_atomic_addzScheduler._any_atomic_addÎ  s   € Üñ 
ð
 ô
ó 
ð 	
rn   c                ó"  — | j                  |d|¬«      }t        j                  |«      }t        j                  j
                  j                  «       }|j                  «       sd }||fS |j                  d|¬«      }t        |t        «      sJ ‚||fS )NT)r{  rz  Útriton_)Úkernel_nameÚsource_code)rx  r   ÚloadrŸ  r   Úasync_compileÚAsyncCompileÚuse_process_poolrÒ   rŒ   r   )rk   rÒ  rz  Úsrc_codeÚmodr¸  Úfuts          rl   Úcompile_kernelzScheduler.compile_kernel×  s˜   € ð ×7Ñ7Ø D¸ð 8ó 
ˆô ×Ñ˜xÓ(ˆÜŸ™×5Ñ5×BÑBÓDˆØ×-Ñ-Ô/ØˆCð
 �SˆzÐð  ×&Ñ&°9È(Ð&ÓSˆCÜ˜c¤<Ô0Ð0Ð0à�SˆzÐrn   c                ód  ‡ ‡‡‡‡‡‡‡‡‡ ‡!‡"‡#‡$‡%‡&‡'‡(‡)‡*‡+— t        d„ ‰‰fD «       «      }t        j                  s|st        j	                  d«      S ‰j                  «       r(t        ‰j                  «       t        j                  «      r ‰j                  «       s‰j                  «       rt        j	                  d«      S ‰j                  «       }|d   j                  «       Š‰sJ ‚‰j                  dk(  r(t        j                  dk7  rt        j	                  d«      S ‰j                  «       }t        t!        j"                  ||«      «      }‰ j%                  |«      rt        j	                  d«      S ddlmŠ t+        ‰‰«      Š+|d   j                  «       Š‰€J ‚dˆˆfd„Š%|�rt        d	„ ‰‰fD «       «      �r
‰j                  «       duŠ‰r‰j                  «       n‰j                  «       Š*t        ‰*t        j,                  «      sJ ‚‰ j/                  ‰*«      rt        j	                  d
«      S i Š$g Š"t        j0                  D �]A  }‰*j3                  |«      Št5        ‰j7                  «       d„ ¬«      D ]~  \  }}	t        |t8        j:                  j<                  j>                  «      sŒ5‰*jA                  |«      5  ‰"jC                  |g‰ jE                  ||jF                  ¬«      ¢­«       ddd«       Œ€ tI        d«      }
d}i }‰"D ]V  \  }}}	 |�|jK                  «        ‰*jA                  |«      5  ‰ jY                  |‰«      \  }}|||<   ||
k  r|}
|}ddd«       ŒX |‰*jZ                  |<   t        |t\        «      sJ ‚|‰$|<   �ŒD t        j^                  Šta        d„ ‰*jb                  D «       «      }te        «       xr ‰ xr |t        jf                  k  Š#tI        d«      tI        d«      cŠ'Š(dŠ&‰#sR‰*j3                  «       Š‰*ji                  «       \  Š&Š't5        ‰j7                  «       tk        jl                  d«      ¬«      }n‰*jb                  D �cg c]  }|df‘Œ }}‰r(‰r‰ jo                  |«      n‰ jo                  |«      \  Š(}n4‰st        j	                  d
«      S ‰jq                  «       Š(ts        ‰‰‰(«      Š)ddl:m;} g Š"d}|D ]¤  \  }}t        |t\        «      sŒ‰s&ty        |d«      r|jz                  ‰*jz                  k7  rŒ?‰r
|‰'‰(z   k\  r n\|dz  }|t        jf                  kD  r nB‰*jA                  |«      5  	 ‰"jC                  |g‰ jE                  |«      ¢­«       	 ddd«       Œ¦ t}        ‰"«      dk(  rt        j	                  d
«      S dˆˆˆˆˆ"ˆ#ˆ$ˆ%ˆ&ˆ'ˆ(ˆ)ˆ*ˆ fd„}t        j                  |‰"d   d   «      S ‰ jE                  |«      Š‰ jE                  |«      Š!‰ jE                  |«      Š dˆˆˆˆ ˆ!ˆ%ˆ ˆ+fd„}t        j                  |‰ d   ¬«      S # 1 sw Y   �Œ³xY w# tL        $ rI}tN        jQ                  tR        jT                  «      rtN        jW                  d‰sdnd|«       Y d}~�Œqd}~ww xY w# 1 sw Y   �ŒƒxY wc c}w # |$ r Y ddd«       �Œ×w xY w# 1 sw Y   �ŒåxY w)ú‡
        If config.benchmark_fusion is False, always return True.
        Otherwise, return True if fusion can brings speedup.
        c              3  ó�   K  — | ]>  }|j                  «       xr( t        |j                  «       t        j                  «      –— Œ@ y ­wrw   )r   rŒ   rÆ  r+   r�  r  s     rl   r”   z.Scheduler.speedup_by_fusion.<locals>.<genexpr>ï  sE   è ø€ ò  
ð ð �M‰M‹Oò JÜ˜1×.Ñ.Ó0´"×2HÑ2HÓIóJñ 
ùs   ‚AATr   r  rÒ   ©ÚCompilationErrorNc           
     ót  •— t         j                  t        j                  «      r”| ||z   k  rFt         j	                  d‰j                  «       ‰j                  «       t        ||z   | z  d›«      «       y t         j	                  d‰j                  «       ‰j                  «       t        | ||z   z  d›«      «       y y )Nz9can fuse (benchmark): fusing %s with %s cause %sx speedupú.3fz=cannot fuse (benchmark): fusing %s with %s cause %sx slowdown)r  rÓ  rÔ  ÚDEBUGr  r  rE   rF   )Úms_fusedr  r  r�   r‚   s      €€rl   Ú
log_fusionz/Scheduler.speedup_by_fusion.<locals>.log_fusion  s¦   ø€ Ü×&Ñ&¤w§}¡}Ô5Ø˜c C™iÒ'Ü×$Ñ$ØSØ×.Ñ.Ó0Ø×.Ñ.Ó0Ü" s¨S¡y°HÑ&<¸SÐ%AÓCõ	ô ×$Ñ$ØWØ×.Ñ.Ó0Ø×.Ñ.Ó0Ü  H°°c±	Ñ$:¸3Ð#?ÓAõ	ð 6rn   c              3  ó@   K  — | ]  }|j                  «       d u–— Œ y ­wrw   rÉ  r  s     rl   r”   z.Scheduler.speedup_by_fusion.<locals>.<genexpr>)  s#   è ø€ ò %
Ø23ˆA×ÑÓ!¨Ô-ñ%
ùó   ‚Fc                ó   — | d   S r[  r~   rï  s    rl   rv  z-Scheduler.speedup_by_fusion.<locals>.<lambda>=  s   € ÈaÐPQÉd€ rn   r$  ry  ÚinfúException in compiling %s: %srÔ  rÖ  c              3  ó<   K  — | ]  }t        |t        «      –— Œ y ­wrw   )rŒ   r   )r’   r\  s     rl   r”   z.Scheduler.speedup_by_fusion.<locals>.<genexpr>h  s   è ø€ ò %Ø<=”
˜1Ô6×7ñ%ùrý  r   )Ú	CantSplitÚallowed_prologue_inpsc                 óö  •— t        d«      } d }i }‰rh‰rt        ‰t        j                  «      sJ ‚‰j	                  «       Š‰j                  «       \  ŠŠ‰D �cg c]  }|d   ‰v r|‘Œ c}Št        ‰ˆfd„¬«      Š‰D �]0  \  }}}	 |�|j                  «       }n!‰s|j                  }|j                  «        nd }‰r>‰j!                  |«      5  ‰j#                  |‰«      \  }	}
|	||<   |	| k  r|	} |}d d d «       Œ}‰|k(  xs ‰‰z   ‰|   ‰z   kD  }|sŒ•|sŒ˜|j                  «        |j$                  r|j&                  sJ ‚|j$                  d   }|j&                  }|j(                  }t+        ‰‰|j&                  |||j,                  j.                  t1        j2                  ‰«      «      }|s�Œ/|} n ‰r
 ‰| ‰‰«       ‰r| ‰‰z   k  rL|�Jt4        j6                  r|‰d <   ‰j9                  ‰«       n‰j;                  |«       ‰r|‰j<                  d <   yy	c c}w # t        $ rI}t        j                  t        j                  «      rt        j                  d‰sdnd|«       Y d }~�Œéd }~ww xY w# 1 sw Y   �ŒûxY w)
NrÌ  r   c                ó   •— ‰| d      S rÅ   r~   )rJ  r“  s    €rl   rv  zKScheduler.speedup_by_fusion.<locals>.benchmark_when_ready.<locals>.<lambda>Ë  s   ø€  n°Q°q±TÑ&:€ rn   r$  rÍ  rÔ  rÖ  TF)r¯  rŒ   r+   r�  r“  r’  r)  r-  r´  Ú
precompiler’  r  rÓ  rÔ  rÆ  r  Úswap_as_triton_callerr}  Ú	launchersÚn_regsÚn_spillsr  Úbmreqr  rC   rÆ  r(   r”  r•  r–  Ú_choice_timings)Úmin_ms_fusedÚms_fused_choiceÚnew_timingsÚ
fut_choicerž  rh   Ú	mod_fusedÚresr  rÇ  ÚpathÚfusible_choiceÚcompiled_kernelr  r  Úshould_fuse_epilogueÚbench_epiloguer“  r  r:  Úfuture_choicesÚget_choice_timings_asyncÚ hint_override_best_fusion_choicerÈ  Ú
min_choicer  r  Ú	ms2_fusedr…  rk   s                   €€€€€€€€€€€€€€rl   Úbenchmark_when_readyz9Scheduler.speedup_by_fusion.<locals>.benchmark_when_ready·  s¶  ø€ ä$ U›|�Ø"&�Ø �á+Ù%¬*°ZÄ×AWÑAWÔ*XÐXÐXØ%/×%>Ñ%>Ó%@�NØ&0×&?Ñ&?Ó&A‘O�J ð
 +9ö&à&Ø% a™=¨NÑ:ò #ò&�Nô &,Ø&Û:ô&�Nð
 2@ó <&Ñ-�F˜F Ið!Ø!Ð-Ø"(§-¡-£/™CÙ!/Ø"+×"3Ñ"3˜CØŸN™NÕ,à"&˜Cñ &à'×=Ñ=¸fÓEñ 	9Ø-1×-LÑ-LØ )à &ó.™N˜H dð
 3;˜K¨Ñ/Ø'¨,Ò6Ø/7 Ø28 ÷	9ð 	9ð '¨&Ñ0ò NØ" S™y¨>¸&Ñ+AÀIÑ+MÑMð 'ò
 ¢>Ø"×-Ñ-Ô/à#&§=¢=°V·]²]ÐBÐ#Bà.1¯m©m¸AÑ.>˜Oà+:×+AÑ+A˜Là-<×-EÑ-E˜NÜ3AØ #Ø #Ø &§¡Ø ,Ø .Ø &§¡× 6Ñ 6Ü 0× 7Ñ 7¸Ó ?ó4Ð0ó  4Ø28 Ù %ðy<&ñ| "Ù˜|¨S°#Ô6ñ '¨,¸#À¹)Ò*DØ%Ð1Ü×0Ò0ØAPÐ8¸Ñ>à"×=Ñ=Ø<õð
 #×<Ñ<¸_ÔMá%à;F˜
×2Ñ2°4Ñ8Øà ùò}&øô. %ò !Ü%×2Ñ2´7·=±=ÔAÜ&×,Ñ,Ø ?Ù2A¡
ÀzØ !ôõ
 !ûð!ú÷	9ñ 	9ús*   ÁHÂ4HÃ$I.È	I+È">I&É&I+É.I8	c                 óâ  •‡‡‡‡‡‡— ddl m}  	 ‰d   ‰d   ‰d   fD ]  }|€Œ|j                  «        Œ ‰j                  ‰d   ‰
«      \  ŠŠt	        j
                  ‰«      r	 ‰d«       y‰j                  ‰d   ‰
«      \  ŠŠt	        j
                  ‰«      r	 ‰d«       y‰j                  ‰d   ‰
«      \  ŠŠt	        j
                  ‰«      r	 ‰d«       y ‰‰‰‰«       t        d«      rW‰‰‰z   k\  rO‰‰f‰j                  vr?‰j                  j                  ‰‰f«       t        d«      j                  ˆˆˆˆˆˆfd	„«       ‰‰‰z   k  S # | $ r Y y‰	$ r}d
t        |«      v rY d }~y‚ d }~ww xY w)Nr   )ÚNoTritonConfigsErrorr   z%register spilling of the first kernelFz&register spilling of the second kernelz%register spilling of the fused kernelÚslow_fusionc            	     ó$   •— ‰‰ ‰‰‰‰‰‰ ‰z   z  dœS )N)Úkernel1_pathÚkernel1_latencyÚkernel2_pathÚkernel2_latencyÚfused_kernel_pathÚfused_kernel_latencyÚslow_down_ratior~   )r  r  rÇ  Úpath1Úpath2Ú
path_fuseds   €€€€€€rl   rv  zKScheduler.speedup_by_fusion.<locals>.benchmark_when_ready.<locals>.<lambda>`  s(   ø€ Ø05Ø36Ø05Ø36Ø5?Ø8@Ø3;¸sÀS¹yÑ3Iñ%€ rn   úLoop-carried variableT)Ú)torch._inductor.runtime.triton_heuristicsrì  r-  r}  rB  Úisinfr    r‚  rÜ  r   rª  r  )rì  r½  r  r  r  rÇ  rö  r÷  rø  rÃ  r  Úfuture_and_mod_l1Úfuture_and_mod_l1_fusedÚfuture_and_mod_l2rÈ  rk   r  s      @@@@@@€€€€€€€€rl   rê  z9Scheduler.speedup_by_fusion.<locals>.benchmark_when_ready-  sœ  þ€ õðAð *¨!Ñ,Ø)¨!Ñ,Ø/°Ñ2ð ò )˜ð
 ™?ØŸJ™J�Lð)ð "&×!@Ñ!@Ø)¨!Ñ,àó"‘J�C˜ô
 —z‘z #”ÙÐCÔDØ$à!%×!@Ñ!@Ø)¨!Ñ,àó"‘J�C˜ô
 —z‘z #”ÙÐDÔEØ$à+/×+JÑ+JØ/°Ñ2àó,Ñ(�H˜jô
 —z‘z (Ô+ÙÐCÔDØ$á˜x¨¨cÔ2ô 0°Ô>Ø$¨¨c©	Ò1Ø" E˜N°$×2IÑ2IÑIà×/Ñ/×3Ñ3°U¸E°NÔCÜ(¨Ó7×?Ñ?÷ð ô
ð $ c¨C¡iÑ/Ð/øà+ò !Ù à'ò Ø.´#°a³&Ñ8Ü#Øûðús<   �E £AE Á+5E Â!5E ÃA3E ÅE.ÅE.ÅE)Å(E)Å)E.rt   )rÇ  r¯  r  r¯  r  r¯  r†   rU  rV  )@ræ   r(   Úbenchmark_fusionrd   rr   r   rŒ   rÆ  r+   ÚTritonTemplateBufferr$  r–   rÞ   r¶   rj  rÿ   rÔ  rÕ  r²  Útriton.compiler.errorsrÃ  r  r�  rŠ  r”  r“  r)  r�  rŸ  r   rŽ  ÚTritonTemplateCallerrÔ  rÀ   r¾  rz  r¯  r-  r’  r  rÓ  rÔ  rÆ  r  r}  rÙ  r   Úbenchmark_epilogue_fusionrÙ   Úchoicesr   Ú max_epilogue_benchmarked_choicesr’  ÚoperatorÚ
itemgetterrt  r¥  r  rc  rÏ  r<  rÐ  r©   ru   ),rk   r�   r‚   Úis_multi_templateÚnode_list_1Únode_list_2Únode_list_fusedrz  rž  r  rÚ  rÛ  rÜ  rh   rÞ  r  rÇ  rà  Únum_triton_callersÚchoice_timings_iterr\  r÷  rÏ  Útriton_choicesÚunfused_timerê  rÃ  rä  r“  r  r:  rü  rý  rþ  rå  ræ  rç  rÈ  rè  r  r  ré  r…  r  s,   ```                       @@@@@@@@@@@@@@@@@@rl   Úspeedup_by_fusionzScheduler.speedup_by_fusionç  sÕ  ÿÿü€ ô  ñ  
ð ˜U�^ô 
ó 
Ðô
 ×&Ò&Ñ/@Ü×$Ñ$ TÓ*Ð*ð ×ÑÔÜ˜u×6Ñ6Ó8¼"×:QÑ:QÔRØ×ÑÔ!Ø×ÑÔ!ô  ×$Ñ$ TÓ*Ð*à—o‘oÓ'ˆØ˜Q‘×*Ñ*Ó,ˆÙˆˆvð �;‰;˜%Ò¤F×$6Ñ$6¸(Ò$BÜ×$Ñ$ TÓ*Ð*à—o‘oÓ'ˆÜœyŸ™¨{¸KÓHÓIˆð
 ×Ñ Ô0Ü×$Ñ$ TÓ*Ð*å;ä˜˜uÓ%ˆà  Ñ#×.Ñ.Ó0ˆØÐ!Ð!Ð!ö	ò" ¤ñ %
Ø8=¸u°~ô%
õ "
ð $×5Ñ5Ó7¸tÐCˆOñ #ð ×'Ñ'Ô)à×,Ñ,Ó.ð ô
 ˜j¬"×*@Ñ*@ÔAÐAÐAà×5Ñ5°jÔAÜ#×(Ñ(¨Ó/Ð/ð ð -ð QSˆNÜ!'×!:Ñ!:ó *R�Ø!+×!:Ñ!:¸=Ó!I�Ü!'¨×(<Ñ(<Ó(>ÁNÔ!Sò ‘I�F˜AÜ%Ø¤§¡× @Ñ @× UÑ Uôð !Ø#×9Ñ9¸&ÓAñ Ø&×-Ñ-à &ðà!%×!4Ñ!4Ø$3À6×CWÑCWð "5ó ""ñô÷ð ðô  % U›|�ØCG�Ø �Ø1?ò 5Ñ-�F˜F Ið
!Ø!Ð-Ø"ŸM™MœOð $×9Ñ9¸&ÓAñ 5Ø)-×)HÑ)HØ% vó*™˜ $ð /7˜ FÑ+Ø# lÒ2Ø+3˜LØ.4˜O÷5ð 5ð5ð( =H�
×*Ñ*¨=Ñ9Ü! /Ô3KÔLÐLÐLØBQÐ0°Ó?ðU*RôX $×=Ñ=ˆNÜ!$ñ %ØAK×ASÑASô%ó "Ðô
 )Ó*ò RØ&Ð&òRà&¬&×*QÑ*QÑQð %ô ˜U“|¤U¨5£\ˆHˆC�Ø15ˆJÙ+à!+×!:Ñ!:Ó!<�Ø",×";Ñ";Ó"=‘�
˜CÜ&,Ø"×(Ñ(Ó*´×0CÑ0CÀAÓ0Fô'Ñ#ð 8B×7IÑ7IÖ&J°!¨¨1¢vÐ&JÐ#Ð&Jáñ 'ð ×.Ñ.¨{Ô;à×3Ñ3°KÓ@ñ �‘Uñ 'Ü'×,Ñ,¨UÓ3Ð3à×2Ñ2Ó4�Ü<¸UÀEÈ3ÓO�	õ ?àPRˆNØˆNØ(;ò !Ñ$�˜Ü! &Ô*BÔCØñ (Ü Ð(?Ô@Ø×4Ñ4¸
×8XÑ8XÒXàá! l°c¸C±iÒ&?Ùà !Ñ#�Ø!¤F×$KÑ$KÒKÙà×5Ñ5°fÓ=ñ !ð!Ø&×-Ñ-Ø#ÐK d×&9Ñ&9¸/Ó&JÑKõ÷!ð !ð/!ôB �>Ó" aÒ'Ü#×(Ñ(¨Ó/Ð/÷j!÷ j!ò j!ôX  ×-Ñ-Ø$ n°QÑ&7¸Ñ&:óð ð !%× 3Ñ 3°KÓ @ÐØ $× 3Ñ 3°KÓ @ÐØ&*×&9Ñ&9¸/Ó&JÐ#÷Fô FôP  ×-Ñ-Ø0Ð9PÐQRÑ9Sð .ó ð ÷g	ñ ûô" %ò !Ü%×2Ñ2´7·=±=ÔAÜ&×,Ñ,Ø ?Ù2A¡
ÀzØ !ôõ
 !ûð!ú÷5ñ 5üòF 'Køðb %ò !ð !÷!ñ !ð
!ú÷!ñ !ús`   Ê:1XÌX*Ì1$Y?ÑZÕ Z%Õ$ZØX'Ø*	Y<Ø3>Y7Ù7Y<Ù?Z	ÚZ"ÚZ%Ú!Z"Ú"Z%Ú%Z/	c                ó<   — | j                   |j                  «          S )z0Look up the node in Scheduler name_to_fused_node)ró  rù  r%  s     rl   r2  zScheduler.get_fused_nodey  s   € à×&Ñ& t×':Ñ':Ó'<Ñ=Ð=rn   c                óL  — t         j                  d|j                  «       |j                  «       «       |j                  «       }|j                  «       |k(  sJ ‚| j	                  |«      j                  ||«      }|j                  |«       |j                  |«       |j                  |«       | j                  j                  |j                  «       D �ci c]  }|j                  «       |“Œ c}«       | j                  j                  |«      }|�|| j                  |<   |S c c}w )Nzfusing %s with %s)r  r  r  rÞ   rc  rr   r\  rÜ  ró  r  r–   r¡  rÈ  )rk   r�   r‚   rˆ  r  Únode3r  Ústream1s           rl   Úfuse_two_nodeszScheduler.fuse_two_nodes}  sù   € ô 	×ÑÐ,¨e¯n©nÓ.>ÀÇÁÓ@PÔQà×!Ñ!Ó#ˆØ×ÑÓ! VÒ+Ð+Ð+Ø× Ñ  Ó(×-Ñ-¨e°UÓ;ˆØ×Ñ˜5Ô!Ø×Ñ˜5Ô!Ø�‰˜ÔØ×Ñ×&Ñ&ÀUÇ_Á_ÓEVÖ'WÀ¨¯
©
«°eÑ(;Ò'WÔXð ×%Ñ%×)Ñ)¨%Ó0ˆØÐØ)0ˆD×Ñ Ñ&àˆùò (Xs   ÃD!c                ó‚   — | j                  ||«      r-| j                  ||«      s |«       r| j                  |||«       yy©NTF)rî   Úwill_fusion_create_cycler  )rk   r�   r‚   Ú
speedup_fnrˆ  s        rl   Úfuse_if_speedupzScheduler.fuse_if_speedup•  s?   € ð �M‰M˜% Ô'Ø×1Ñ1°%¸Ô?Ù”à×Ñ  u¨kÔ:Øàrn   c                óˆ  — |�r¿g }i }t        «       }|D �]  }||v rt        ||   «      dk\  sJ ‚||   j                  d«      }t        ||   «      dk(  r|j                  |«       |j	                  «       \  }}	|	|k(  rt        ||	«      sJ ‚|}
n||k(  sJ ‚t        ||	«      sJ ‚|	}
| j                  |
«      |
urŒ¥|j                  r3|j                  j                  }|€J ‚|j                  |«       ||f||<   Œä| j                  ||	|j                  |«      s�Œ|j                  |«       �Œ t        |«      D ]l  }||   \  }}| j                  | j                  |j                  «      | j                  |j                  «      |j                  |«      sŒ\|j                  |«       Œn |D ]  }|j                  |«       Œ |r�Œ¾yy)zÁ
        Evaluate pending template fusions for a set of fusion candidate nodes.
        The fusion candidate nodes are pointwise nodes as potential epilogue
        or prologue fusions
        r   r   N)r   r©   rÛ  rÜ  r…   rD  rG  r2  rh   rÀ   r  rf   r   r�   r‚   )rk   Útemplate_fusion_candidatesrˆ  Útemplate_futuresÚfuture_to_pending_fusionÚfusions_to_removeÚ	candidateÚpending_fusionr�   r‚   rÕ  ÚfÚcands                rl   Ú"_evaluate_pending_template_fusionsz,Scheduler._evaluate_pending_template_fusions¦  s  € ò )Ø-/Ðð ð %ô @J»|ÐØ7ó %9�	àÐ!;Ñ;ÜÐ6°yÑAÓBÀaÒGððHð "<¸IÑ!F×!JÑ!JÈ1Ó!M�äÐ1°)Ñ<Ó=ÀÒBØ%×)Ñ)¨)Ô4à-×>Ñ>Ó@‘��uà˜IÒ%Ü-¨e°UÔ;Ð;Ð;Ø$)‘Mà  IÒ-Ð-Ð-Ü-¨e°UÔ;Ð;Ð;Ø$)�Mð ×&Ñ& }Ó5¸]ÑJØà!×(Ò(Ø&×-Ñ-×4Ñ4�AØ˜=Ð(˜=Ø$×+Ñ+¨AÔ.Ø3AÀ9Ð2MÐ,¨QÒ/ð ×+Ñ+Ø˜u n×&@Ñ&@À+öð *×-Ñ-¨iÖ8ðK%9ôP "Ð"2Ó3ò 0�Ø'?ÀÑ'BÑ$� Ø×'Ñ'Ø×'Ñ'¨×(<Ñ(<Ó=Ø×'Ñ'¨×(<Ñ(<Ó=Ø"×.Ñ.Øõ	ð &×)Ñ)¨$Õ/ð0ð 'ò 2�Ø*×.Ñ.¨qÕ1ð2õq )rn   c                óÂ  ‡ ‡‡— 	 	 	 	 	 	 dˆˆˆ fd„}|D �]H  \  }} |||«       ‰ j                  |«      }‰ j                  |«      }t        ||«      r||f‰ j                  v rŒO‰ j                  |||«      sŒc‰ j	                  ||«      rŒv‰ j                  ||«      }	|	j                  �”t        |	j                  |||	j                  ¬«      }
t        ||«      rY||f‰ j                  vsJ ‚‰ j                  j                  ||f«       t        ||«      }||vrg ||<   ||   j                  |
«       n
|
‰|<   |
‰|<   �Œ(|	j                  s�Œ6‰ j                  ||‰«       �ŒK y )Nc                ób  •— ‰j                  | «      ‰v s‰j                  |«      ‰v �r‰j                  ‰j                  | «      ‰j                  ‰j                  |«      «      «      }|€J ‚|j                  «       \  }}|j                  }‰j	                  |d «       ‰j	                  |d «       ‰j                  |«      |u sJ ‚‰j                  |«      |u sJ ‚ |«       r‰j                  | |«      rŒñ‰j                  ||‰«       ‰j                  | «      ‰v rŒñ‰j                  |«      ‰v r�Œy y rw   )r2  rÈ  r…   rf   rÛ  r  r  )	r�   r‚   r!  Ú	node_key1Ú	node_key2Ú
is_speeduprˆ  Úpending_fusionsrk   s	         €€€rl   Úresolve_pending_fusionsz<Scheduler._try_fusion_pairs.<locals>.resolve_pending_fusionsô  s1  ø€ ð
 ×#Ñ# EÓ*¨oÑ=Ø×&Ñ& uÓ-°Ò@à!0×!4Ñ!4Ø×'Ñ'¨Ó.Ø#×'Ñ'¨×(;Ñ(;¸EÓ(BÓCó"�ð &Ð1Ð1Ð1à'5×'FÑ'FÓ'HÑ$�	˜9Ø+×7Ñ7�
à×#Ñ# I¨tÔ4Ø×#Ñ# I¨tÔ4à×*Ñ*¨9Ó5¸ÑBÐBÐBØ×*Ñ*¨9Ó5¸ÑBÐBÐBá!”| t×'DÑ'DÀUÈEÔ'RØà×#Ñ# I¨y¸+ÔFð+ ×#Ñ# EÓ*¨oÒ=Ø×&Ñ& uÓ-°Õ@rn   )rf   r�   r‚   rh   r"  )r2  rI  ry  rî   r  r  rf   r€   rh   rÜ  rK  rÀ   re   r  )rk   Úpossible_fusion_pairsr*  Útemplate_fusion_nodesrˆ  rl  r+  r�   r‚   Ú
fusion_resr!  Útemplate_pw_nodes   ` ` `       rl   Ú_try_fusion_pairszScheduler._try_fusion_pairsì  s”  ú€ ð	GØ$ð	Gà$ð	Gð ÷	Gð8 2ó +	?‰LˆE�5ñ $ E¨5Ô1Ø×'Ñ'¨Ó.ˆEØ×'Ñ'¨Ó.ˆEô # 5¨%Ô0Ø˜E�N d×&@Ñ&@Ñ@àà�}‰}Ø�uÐ.õà×3Ñ3°E¸5ÕAØ!×3Ñ3°E¸5ÓA�
Ø×)Ñ)Ð5Ü%2Ø$.×$:Ñ$:Ø#Ø#Ø)×0Ñ0ô	&�Nô *¨%°Ô7Ø % u˜~°T×5OÑ5OÑOÐOÐOØ×2Ñ2×6Ñ6¸¸u°~ÔFä+BÀ5È%Ó+PÐ(Ø+Ð3HÑHØFHÐ1Ð2BÑCØ-Ð.>Ñ?×FÑFÀ~ÕVà1?˜¨Ñ.Ø1?˜¨Ñ.áà!×-Ò-Ùà×#Ñ# E¨5°+Ö>ñW+	?rn   c                ó@  — t        «       }|j                  «       D ]�  }|j                  «       \  }}|j                  }||v st	        ||«      rŒ3|j                  |«       | j                  |«      |u sJ ‚| j                  |«      |u sJ ‚| j                  ||||«       Œƒ y rw   )r   r¹   r…   rf   rI  rÜ  r2  r  )rk   rˆ  r*  Úseen_pair_speedup_fnr!  r'  r(  Úis_speedup_fns           rl   Ú_finish_pending_fusionsz!Scheduler._finish_pending_fusions=  s·   € ô
 @J»|Ðð .×4Ñ4Ó6ò 	SˆNØ#1×#BÑ#BÓ#DÑ ˆI�yØ*×6Ñ6ˆMàÐ 4Ñ4Ô8JØ˜9ô9ð à ×$Ñ$ ]Ô3à×&Ñ& yÓ1°YÑ>Ð>Ð>Ø×&Ñ& yÓ1°YÑ>Ð>Ð>à× Ñ  ¨I°}ÀkÕRñ	Srn   c           
     óî   — t        |D ��cg c]  \  }}t        ||«      sŒ|‘Œ c}}«      }g }|D ]<  \  }}t        ||«      r||v r|j                  ||f«       Œ*|j                  ||f«       Œ> |S c c}}w rw   )r   rD  rG  rÀ   )rk   Úpossible_fusionsÚdeferred_prologue_fusionsÚn1Ún2Úepilogue_template_nodesÚnew_possible_fusionss          rl   Ú_handle_template_overlapz"Scheduler._handle_template_overlapU  sŠ   € ô #-Ø.×M‘F�B˜Ô2DÀRÈÕ2LŠRÓMó#
Ðð  "ÐØ&ò 	6‰FˆB�Ü! " bÔ)¨bÐ4KÑ.KØ)×0Ñ0°"°b°Õ:à$×+Ñ+¨R°¨HÕ5ð		6ð $Ð#ùó Ns
   ‹A1
 A1
c                óº  — | j                  |«       t        |«      }t        j                  t        j
                  «      r@t        j                  d«       |D ]&  }t        j                  d|j                  «       «       Œ( i }i }g }| j                  ||«      }t        j                  st        j                  r| j                  ||«      }| j                  |||||«       | j                  ||«       | j                  ||«       |j!                  «        |r'| j                  |||||«       | j                  ||«       t#        |d„ ¬«      }| j%                  |«      }|S )a  
        Combine eligible nodes into FusedSchedulerNodes.

        This relies on two key functions to control the logic:
            - self.can_fuse(): checks if a fusion is legal
            - self.score_fusion(): assigns priority to a given fusion
        zfuse_nodes_once, candidates:z  %sc                ó   — | j                   S rw   rå  rï  s    rl   rv  z+Scheduler.fuse_nodes_once.<locals>.<lambda>ª  s
   € °!·+±+€ rn   r$  )rô  r   r  rÓ  rÔ  rÆ  r  r§  Úget_possible_fusionsr(   rŒ  r�  r<  r0  r4  r$  Úclearr)  r{  )	rk   rÒ  rl  rˆ  r�   r*  r-  r7  r6  s	            rl   rm  zScheduler.fuse_nodes_oncei  sk  € ð 	×!Ñ! %Ô(Ü  Ó'ˆÜ×"Ñ"¤7§=¡=Ô1Ü×ÑÐ;Ô<Ø#ò A�Ü× Ñ  ¨×)=Ñ)=Ó)?Õ@ðAð ð 	ð
 OQÐð ð 	"ð  ×4Ñ4ØØó
Ðô
 ×#Ò#¤v×':Ò':Ø#×<Ñ<Ø Ð";ó Ðð 	×ÑØØØ!ØØô	
ð 	×$Ñ$ [°/ÔBà×/Ñ/Ð0EÀ{ÔSØ×#Ñ#Ô%á$Ø×"Ñ"Ø)ØØ%ØØ ôð ×3Ñ3Ð4IÈ;ÔWä�{Ñ(=Ô>ˆØ×.Ñ.¨uÓ5ˆØˆrn   c                óh  — t        | j                  «      }d}t        | j                  «      }t        j	                  d|«       t        t        j                  | «      «      D �]P  \  }}t        j                  |«      }t        |«      dk  rŒ+|�||kD  r �n| j                  |«      st        j	                  d|«       Œ]|dz  }t        j                  dkD  }t        |d   j                  |d|¬«      }t        j                  d	t        |«      |«       |D ]  }	|j                  |	«       Œ |j                  |«       | j                   j#                  |j%                  «       D �
ci c]  }
|
j'                  «       |“Œ c}
«       | j(                  j+                  |d   «      }|€�ŒB|| j(                  |<   �ŒS t-        |d
„ ¬«      | _        | j/                  | j                  «      | _        t        j                  d||t        | j                  «      «       | j1                  | j                  «       yc c}
w )z'
        Groups parallel nodes
        r   z2ComboKernels: Generating with num_ck_nodes = %s...r¨   Nz)ComboKernels: Not speeding up %d-th groupr   TrÛ  z0ComboKernels: Combining %d nodes for %d-th groupc                ó   — | j                   S rw   rå  rï  s    rl   rv  z5Scheduler.create_combo_kernel_nodes.<locals>.<lambda>Ù  s
   € °q·{±{€ rn   r$  zDGenerated ComboKernel nodes: %d ComboKernels, totally %d -> %d nodes)r   rÒ  r©   r“  r  rÑ  ro  r³  rœ  Úspeedup_by_combo_kernelr(   rÝ  r  r©  r\  rÜ  ró  r  r–   r  r¡  rÈ  r)  r{  rô  )rk   rX  rˆ  rn  Únum_nodes_origÚnumr±  r‡  rÝ  r�   r  Ústreams               rl   r’  z#Scheduler.create_combo_kernel_nodes®  sô  € ô ! §¡Ó,ˆØˆÜ˜TŸZ™Z›ˆÜ�	‰	ÐFÈÔUÜ'Ü&×DÑDÀTÓJó
ó "	:‰NˆC�ô 3×CÑCÀIÓNˆIÜ�9‹~ Ò!ØØÐ'¨E°LÒ,@ÚØ×/Ñ/°	Ô:Ü—	‘	ÐEÀsÔKØØ�Q‰JˆEÜ$×;Ñ;¸aÑ?ˆOÜ4Ø˜!‘×&Ñ&ØØ*.Ø /ô	ˆKô �H‰HØBÜ�I“Øôð
 "ò )�Ø×"Ñ" 4Õ(ð)à�O‰O˜KÔ(Ø×#Ñ#×*Ñ*Ø4?×4IÑ4IÓ4KÖL¨q�—‘“˜{Ñ*ÒLôð
 ×(Ñ(×,Ñ,¨Y°q©\Ó:ˆFØÒ!Ø39�×#Ñ# KÓ0ðE"	:ôF ˜KÑ-BÔCˆŒ
Ø×3Ñ3°D·J±JÓ?ˆŒ
Ü�‰ØRØØÜ�—
‘
‹Oô		
ð 	×!Ñ! $§*¡*Õ-ùò Ms   Å"H/
c                óH   — |D ]  }|j                  | j                  «       Œ y rw   )rô  ró  )rk   rÒ  r�   s      rl   rô  zScheduler.prune_redundant_depsã  s%   € Øò 	?ˆDØ×%Ñ% d×&=Ñ&=Õ>ñ	?rn   c                óÌ  ‡ ‡‡
‡— g Š
t        t        t        t        f      «       Šdˆˆ
ˆˆ fd„}t        j                  t
        «      }|D ]=  }‰ j                  |«      rŒ|j                  «       D ]  }||   j                  |«       Œ Œ? |j                  «       D ]
  } ||«       Œ t        j                  rat        j                  t
        «      }|D ]&  }t        |dd«      }	|	sŒ||	   j                  |«       Œ( |j                  «       D ]
  } ||«       Œ ‰ j                  ‰
«      Š
‰
j                  ‰ j                  d¬«       t         j#                  dt%        ‰
«      «       ‰
S )z^
        Helper to find all legal fusion opportunities, sorted by self.score_fusion()
        c                ó|  •— t        | «      D ]­  \  }}| |dz   |dz   t        j                  z    D ]‰  }||f}|‰v rŒ‰j                  |«       ‰j	                  ||‰«      r‰j                  |«       ŒB|j                  «       s|j                  «       sŒc‰j	                  ||‰«      sŒw‰j                  ||f«       Œ‹ Œ¯ y r[  )rÑ  r(   Ú)max_fusion_buffer_group_pairwise_attemptsrÜ  rî   rÀ   r   r$  )	rÒ  Únode1_indexr�   r‚   r%  rl  r6  rI  rk   s	        €€€€rl   Úcheck_all_pairsz7Scheduler.get_possible_fusions.<locals>.check_all_pairsò  sÒ   ø€ Ü&/°Ó&6ò @Ñ"�˜UØ"Ø !‘O kØñ'ä×FÑFñ'Gðò @�Eð
 ! %˜.�CØ˜d‘{Ø Ø—H‘H˜S”Mà—}‘} U¨EÐ3CÔDØ(×/Ñ/°Õ4Ø×+Ñ+Ô-°×1AÑ1AÕ1CÈÏÉØ˜uÐ&6õJð )×/Ñ/°¸°Õ?ñ!@ñ@rn   r¡   NT)r%  Úreversezfound %d possible fusions©rÒ  ræ  r†   rU  )r   rª   r_   r•  r   rÿ   Úunfusable_noder¿   rÀ   r¹   r(   Úaggressive_fusionr�  Ú*get_possible_fusions_with_highest_priorityrí  Úscore_fusion_keyr  r  r©   )rk   rÒ  rl  rL  Úbuffer_names_groupingr�   rº   Únode_groupingÚgroup_groupingr¡   r6  rI  s   ` `       @@rl   r?  zScheduler.get_possible_fusionsç  sn  û€ ð ÐÜœ%Ô 1Ô3DÐ DÑEÑFÓHˆ÷	@ð 	@ô( !,× 7Ñ 7¼Ó =ÐØò 	8ˆDØ×"Ñ" 4Ô(ØØ×-Ñ-Ó/ò 8�Ø% cÑ*×1Ñ1°$Õ7ñ8ð	8ð
 3×9Ñ9Ó;ò 	+ˆMÙ˜MÕ*ð	+ô ×#Ò#Ü(×4Ñ4´TÓ:ˆNØò 7�Ü  g¨tÓ4�ÚØ" 5Ñ)×0Ñ0°Õ6ð7ð "0×!6Ñ!6Ó!8ò /�Ù Õ.ð/ð  ×JÑJØó
Ðð 	×Ñ $×"7Ñ"7ÀÐÔFÜ×ÑÐ4´cÐ:JÓ6KÔLØÐrn   c                óÒ  ‡ ‡‡‡‡— t        t           «       Šdˆˆˆˆ ˆfd„Š|j                  «       j                  j	                  «       |j                  «       j                  j	                  «       z  Š|j
                  j                  j	                  «       |j
                  j                  j	                  «       z  ‰z
  Št        ˆˆ fd„‰D «       «      }|r t        ||«      d«       |S )z~
        Finds whether there's a path from node1 to node2 (or vice-versa)
        caused indirectly by other fusions.
        c                ó  •— t        | t        «      rq| ‰vrm‰j                  | «       | j                  «       j	                  ‰«      ryt        ‰| j                  z  «      xs" t        ˆˆfd„| j                  ‰z
  D «       «      S y)NFc              3  óH   •K  — | ]  } ‰‰j                   |   «      –— Œ y ­wrw   rS  ©r’   r  Ú
found_pathrk   s     €€rl   r”   zIScheduler.will_fusion_create_cycle.<locals>.found_path.<locals>.<genexpr>:  s+   øè ø€ ò Hàñ # 4×#:Ñ#:¸1Ñ#=×>ñHùó   ƒ")rŒ   rµ   rÜ  rà   Úissubsetrv   rß   ræ   )r�   Úcombined_ancestorsÚcombined_namesrZ  rk   Úvisiteds    €€€€€rl   rZ  z6Scheduler.will_fusion_create_cycle.<locals>.found_path)  s�   ø€ ä˜$Ô 2Ô3¸ÀGÑ8KØ—‘˜DÔ!Ø×+Ñ+Ó-×6Ñ6Ð7IÔJð !ô   °·±Ñ ?Ó@ò ÄCô Hà!%§¡Ð2DÑ!DôHó Eð ð rn   c              3  óH   •K  — | ]  } ‰‰j                   |   «      –— Œ y ­wrw   rS  rY  s     €€rl   r”   z5Scheduler.will_fusion_create_cycle.<locals>.<genexpr>H  s!   øè ø€ ÒW¸q‘J˜t×6Ñ6°qÑ9×:ÑWùr[  zwill create cycler	  )r   rµ   rà   Ú_dictr   rß   ræ   r  )rk   r�   r‚   Úcycler]  r^  rZ  r_  s   `   @@@@rl   r  z"Scheduler.will_fusion_create_cycle  sÇ   ü€ ô Ô/Ñ0Ó2ˆ÷	ñ 	ð2 ×%Ñ%Ó'×-Ñ-×2Ñ2Ó4Ø×'Ñ'Ó)×/Ñ/×4Ñ4Ó6ñ7ð 	ð
 �O‰O×!Ñ!×&Ñ&Ó(¨5¯?©?×+@Ñ+@×+EÑ+EÓ+GÑGØñÐô ÔWÐDVÔWÓWˆÙØ#ŒI�e˜UÓ#Ð$7Ô8Øˆrn   c                ó�  ‡ ‡— ddl mŠ 	 	 	 	 dˆ fd„} ||«      } ||«      }t        ˆfd„|D «       «      }t        ˆfd„|D «       «      }|j                  |«      }d}	|D ]  }
	 |	t	        |
d   «      z  }	Œ ‰ j                  ||«      }t        j                  j                  j                  |	d	|z  «      ry
y# t
        $ r Y  yw xY w)aó  
        Return true if fusing the two nodes can potentially increasing peak memory.

        The implementation is more like a heuristic since we don't really know if we are at peak
        or not when trying to fuse these two nodes. The order of nodes may change later which makes the
        peak memory estimation hard.

        Here is how we decide the LOWER BOUND of extra memory allocation if we fuse these 2 nodes:
        1. find all buffers read by each node with a single user. These buffers are supposed to
           be reused if we don't fuses these 2 nodes
        2. find the intersection of these buffers for the two node and sum the total buffer size.
           If we don't fuse these two nodes, we can at lease avoid this much memory allocation.
           Note that the extra memory allocation is not necessarily causing peak memory increase.
           This is just a heuristic.

        We return true only if the saving for fusion can not trade off the extra memory allocation.
        r   )Úbuffer_reuse_keyc                ó0  •— g }| j                   j                  D ]y  }‰j                  j                  |j                  «      }|sŒ+t        |j                  «      dk(  sŒD|j                  j                  «       sŒ_|j                  |j                  «       Œ{ |S r[  )
r°   r±   r@  rÈ  r²   r©   r  r�   Úhas_tensor_outputrÀ   )r�   r{  rx  rº   rk   s       €rl   Ú_find_single_user_inputszKScheduler.can_fusion_increase_peak_memory.<locals>._find_single_user_inputsd  sw   ø€ ð ˆFØ×&Ñ&×,Ñ,ò ,�Ø×&Ñ&×*Ñ*¨2¯7©7Ó3�Úœ3˜sŸy™y›>¨QÓ.°3·8±8×3MÑ3MÕ3OØ—M‘M #§(¡(Õ+ð,ð ˆMrn   c              3  ó.   •K  — | ]  } ‰|«      –— Œ y ­wrw   r~   ©r’   rº   rd  s     €rl   r”   z<Scheduler.can_fusion_increase_peak_memory.<locals>.<genexpr>r  ó   øè ø€ Ò#S¸cÑ$4°S×$9Ñ#Sùr>  c              3  ó.   •K  — | ]  } ‰|«      –— Œ y ­wrw   r~   ri  s     €rl   r”   z<Scheduler.can_fusion_increase_peak_memory.<locals>.<genexpr>s  rj  r>  r   r¨   Fr  T)r�   r_   r†   zlist[ir.Buffer])r7  rd  r   Úintersectionr  r±  rI  rZ   rœ   r�   Ústatically_known_gt)rk   r�   r‚   rg  Úlhs_dep_nodesÚrhs_dep_nodesÚlhs_reuse_keysÚrhs_reuse_keysÚcommon_reuse_keysÚmemory_overheadr%  Ú	bw_savingrd  s   `           @rl   Úcan_fusion_increase_peak_memoryz)Scheduler.can_fusion_increase_peak_memoryM  sÝ   ù€ õ* 	6ð	Ø#ð	àõ	ñ 1°Ó7ˆÙ0°Ó7ˆä#Ó#SÀ]Ô#SÓSˆÜ#Ó#SÀ]Ô#SÓSˆà*×7Ñ7¸ÓGÐàˆØ$ò 	ˆCðØ¤3 s¨1¡v£;Ñ.‘ð	ð ×,Ñ,¨U°EÓ:ˆ	ô �7‰7×Ñ×/Ñ/°ÀÀiÁÔPØØøô ò âðús   Á$B8Â8	CÃCc                ó¤  — t        |j                  «       D �cg c]  }|j                  «       ‘Œ c}|j                  «       D �cg c]  }|j                  «       ‘Œ c}z   «      }t        d„ |j                  j                  D «       «      }t        d„ |j                  j
                  D «       «      }||z  }t        «       }	|j                  j                  D ]:  }
| j                  |
j                  |«      sŒ |	j                  |
j                  «       Œ< t        d„ |j                  j
                  D «       «      t        d„ |j                  j
                  D «       «      z  }t        d„ |j                  j                  D «       «      t        d„ |j                  j                  D «       «      z  }||z
  }||	z
  }||z  }t        |«      |kD  S c c}w c c}w )Nc              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   zEScheduler.fusion_prevent_too_many_reads_and_writes.<locals>.<genexpr>”  s   è ø€ Ò&T°C s§x¥xÑ&Tùrµ  c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   zEScheduler.fusion_prevent_too_many_reads_and_writes.<locals>.<genexpr>•  ó   è ø€ Ò%R°3 c§h¥hÑ%Rùrµ  c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   zEScheduler.fusion_prevent_too_many_reads_and_writes.<locals>.<genexpr>¢  s   è ø€ ò $
ØˆC�H�Hñ$
ùrµ  c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   zEScheduler.fusion_prevent_too_many_reads_and_writes.<locals>.<genexpr>¤  ry  rµ  c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   zEScheduler.fusion_prevent_too_many_reads_and_writes.<locals>.<genexpr>§  s   è ø€ ò %
ØˆC�H�Hñ%
ùrµ  c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   zEScheduler.fusion_prevent_too_many_reads_and_writes.<locals>.<genexpr>©  s   è ø€ ÒD C�s—x•xÑDùrµ  )
r   r–   r  r°   r�  r±   Ú$can_buffer_be_removed_through_fusionr²   rÜ  r©   )rk   r�   r‚   Ú	thresholdr�   Úfused_node_namesÚnode1_write_namesÚnode2_read_namesÚreads_removed_through_fusionÚwrites_removed_through_fusionrÉ  Úall_read_namesÚall_write_namesÚunique_readsÚunique_writesÚunique_io_bufferss                   rl   Ú(fusion_prevent_too_many_reads_and_writesz2Scheduler.fusion_prevent_too_many_reads_and_writes†  s±  € ô &Ø).¯©Ó):Ö; ˆT�]‰]�_Ò;Ø+0¯?©?Ó+<Ö= 4ˆt�}‰}�Ò=ñ>ó
Ðô 'Ñ&T¸5×;LÑ;L×;SÑ;SÔ&TÓTÐÜ%Ñ%R¸%×:KÑ:K×:QÑ:QÔ%RÓRÐØ'7Ð:KÑ'KÐ$ô :D»Ð%Ø×*Ñ*×1Ñ1ò 	BˆIØ×8Ñ8Ø—‘Ð 0õð .×1Ñ1°)·.±.ÕAð		Bô $ñ $
Ø %× 1Ñ 1× 7Ñ 7ô$
ó 
äÑC¨5×+<Ñ+<×+BÑ+BÔCÓCñDˆô
 %ñ %
Ø %× 1Ñ 1× 8Ñ 8ô%
ó 
äÑD¨5×+<Ñ+<×+CÑ+CÔDÓDñEˆð
 &Ð(DÑDˆð (Ð*GÑGˆð )¨=Ñ8ÐäÐ$Ó%¨	Ñ1Ð1ùòM <ùÚ=s   ˜GÁG
c                ó¤   — t        t        |j                  |j                  z
  «      t        |j                  |j                  z
  «      «      }|dkD  S )aA  
        This function prevents fusion for nodes that can increase memory
        footprint. This problem is more common in horizontal fusion, where nodes
        that are far apart in the original order get fused, lengthening the live
        intervals of tensors. This is very evident in models with activation
        checkpointing, where the recomputed nodes from different checkpointed
        regions get fused and significantly increase the memory footprint.

        The current attempt is a quick, possibly hacky, heuristic to prevent the
        fusion of nodes that are far away in the original order.

        A better but difficult to implement heuristic would be to use live
        intervals of the buffers, find region of peak pressure in the original
        program and prevent fusion that crosses that peak region. We might need
        special care or good approximation in this implementation, as fusion of
        node changes live intervals, and re-computing live intervals and peak
        memory after each fusion can introduce large compilation overhead.
        é@   )r¶  rá  rc  rd  )rk   r�   r‚   Úproximity_scores       rl   Úare_long_distant_nodesz Scheduler.are_long_distant_nodes¶  sE   € ô* Ü�—‘ %§/¡/Ñ1Ó2Ü�—‘ %§/¡/Ñ1Ó2ó
ˆð  Ñ#Ð#rn   c                óþ  — i }|j                   j                  «       D �ci c]  }|j                  |“Œ }}|j                   j                  «       D �ci c]  }|j                  |“Œ }}|D �]}  }t        j                  j                  |«      }	||   }
||   }t        |
t        «      rt        |t        «      sdt        |
«      › dt        |«      › �||<   Œk|
j                  «       |j                  «       k7  r(d|
j                  «       › d|j                  «       › �||<   Œ´t        |
j                  «      t        |j                  «      k7  rd||<   Œå|
j                  «       }|j                  «       }||k7  rd|› d|› �||<   �Œ|
j                  «       |j                  «       k(  rd|
› d|› �||<   �ŒEd}t        |	t        j                  «      sd|	j                   › �}d	|
› d|› d
|› �||<   �Œ€ t#        |«      S c c}w c c}w )z}
        Try to decide reasons why fusion fail due to no shared memory even though
        there are common buffers.
        znot MemoryDep: rš   zdifferent numel: Ú	broadcastzdifferent offset: zMismatch loop orders: r—  zLayout: zUnknown reason: z. )r°   r¶  r²   rZ   rœ   r„  rŒ   r4   r¶   rÌ   rY   r¸   Ú
get_offsetÚnormalize_with_stride_orderr+   r‡  r'  r  )rk   r�   r‚   Úcommon_buf_namesÚreasonsr¼   Únode1_name2depÚnode2_name2depr  rº   Úlhs_depÚrhs_depÚlhs_offÚrhs_offÚ
layout_strs                  rl   Údecide_fusion_fail_reasonz#Scheduler.decide_fusion_fail_reasonÑ  s  € ð ˆØ38×3DÑ3D×3UÑ3UÓ3WÖX¨C˜#Ÿ(™( C™-ÐXˆÐXØ38×3DÑ3D×3UÑ3UÓ3WÖX¨C˜#Ÿ(™( C™-ÐXˆÐXà(ó ,	ˆHÜ—'‘'×$Ñ$ XÓ.ˆCØ$ XÑ.ˆGØ$ XÑ.ˆGä˜g¤yÔ1¼ÀGÌYÔ9Wà%¤d¨7£m _°F¼4À»=¸/ÐJð ˜Ñ!ð à× Ñ Ó" g×&7Ñ&7Ó&9Ò9à'¨×(9Ñ(9Ó(;Ð'<¸FÀ7×CTÑCTÓCVÐBWÐXð ˜Ñ!ð ô ˜WŸ\™\Ó*¬m¸G¿L¹LÓ.IÒIØ$/�˜Ñ!Øà×(Ñ(Ó*ˆGØ×(Ñ(Ó*ˆGØ˜'Ò!ð '9¸¸	ÀÈÀyÐ$Q�˜Ñ!Ùð ×3Ñ3Ó5Ø×6Ñ6Ó8ò9ð '=¸W¸IÀVÈGÈ9Ð$U�˜Ñ!Ùð ˆJÜ˜c¤2×#5Ñ#5Ô6Ø'¨¯
©
 |Ð4�
à" 7 )¨6°'°¸"¸Z¸LÐIð �HÓðU,	ô\ �7‹|Ðùòc YùÚXs   ŸG5ÁG:c                ó´  — t         j                  syt        d„ ||fD «       «      ry|j                  j	                  «       }|j                  j	                  «       }||z  }|syt        d„ |j                  D «       «      }||z
  ryt        |«      dkD  ryt        |j                  j                  «      dkD  s"t        |j                  j                  «      dkD  ryt        t        |j                  j                  «      «      }t        t        |j                  j                  «      «      }t        |t        «      rt        |t        «      sy|j                  j                  D �	ci c]  }	|	j                  |	“Œ }
}	|j                  |
vry|
|j                     }t        |t        «      sy|j                  «       }|j                   |j                   k7  r|j"                  |j"                  k7  ry|j"                  |j"                  k7  st        |j$                  «      dk7  ryt        |j&                  j(                  «      dk7  ry|j&                  j*                  ryd|j&                  j(                  v rd|j&                  j(                  v sJ ‚t        d„ |j&                  j-                  «       D «       «      }t        |«      dk7  ryt        t        |«      «      }||j&                  j(                  d   k(  rd}d}n"||j&                  j(                  d   k(  sJ ‚d}d}d	d
lm} |j&                  j2                  d	   }t        |«      dk7  ryg }t4        j6                  j9                  |«      D ]:  }|j;                  t<        j>                  j@                  jC                  |«      «       Œ< tE        |«      } |||d	   «      }|€y|j&                  j(                  |   |j&                  j(                  |<   ||j&                  j(                  |<   |jG                  dd«       | jI                  ||«      }t        |tJ        «      sJ ‚tL        jO                  d|«       |S c c}	w )aW  
        Attempts to enable fusion between two nodes by inverting indexing patterns.

        This optimization targets cases where node1 has a contiguous write and
        node2 has a contiguous write but discontiguous read. By inverting the
        indexing in node2's read and write operations, we can make them compatible
        with node1 for potential fusion.

        Args:
            node1: First scheduler node (source)
            node2: Second scheduler node (target for inversion)

        Returns:
            int: Fusion score if successful, 0 if optimization not applicable
        r÷   c              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   ©r  r  s     rl   r”   zAScheduler.shared_data_after_inverting_indexing.<locals>.<genexpr>%  s   è ø€ Ò2˜aˆq�x‰x�zÑ2ùrý  c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   zAScheduler.shared_data_after_inverting_indexing.<locals>.<genexpr>1  s   è ø€ ò .
ØˆC�H�Hñ.
ùrµ  r   r¨   Úindex0Úindex1c              3  ó    K  — | ]  }|–— Œ y ­wrw   r~   )r’   rŸ  s     rl   r”   zAScheduler.shared_data_after_inverting_indexing.<locals>.<genexpr>l  s   è ø€ Ò%T¨t¤dÑ%Tùs   ‚r   )Úgenerate_inverse_formulaTFz!Shared memory after inversion: %d)(r(   rn  ræ   r°   Úbuffer_namesr   rq  r©   r±   r�  rÐ  rÈ  rŒ   r4   r²   r`  r³   r¸   Ú	var_namesrù   rþ   Ú	subblocksÚget_read_exprsÚ$torch._inductor.invert_expr_analysisr¤  Úvarsrä   ÚAddÚ	make_argsrÀ   rZ   rœ   r�   Úcombine_modular_indexing_pairsrÙ   rz  rI  r  r  r©  )rk   r�   r‚   Únode1_buffer_namesÚnode2_buffer_namesÚcommon_buffer_namesÚnode2_unmet_dependenciesÚ
node2_readÚnode2_writer¼   Únode1_writesÚnode1_writeÚnode2_read_exprsÚ	read_exprÚread_expr_indexÚwrite_expr_indexr¤  r¬  Úsimplified_termsÚtermÚsimplified_read_exprÚinverse_formulaÚscores                          rl   Ú$shared_data_after_inverting_indexingz.Scheduler.shared_data_after_inverting_indexing  só  € ô& ×4Ò4ØäÑ2 E¨5 >Ô2Ô2Øð #×.Ñ.×;Ñ;Ó=ÐØ"×.Ñ.×;Ñ;Ó=ÐØ0Ð3EÑEÐá"Øô $.ñ .
Ø %× 8Ñ 8ô.
ó $
Ð ð $Ð&8Ò8ØäÐ'Ó(¨1Ò,Øô ˆu× Ñ ×&Ñ&Ó'¨!Ò+¬s°5×3DÑ3D×3KÑ3KÓ/LÈqÒ/PØäœ$˜u×0Ñ0×6Ñ6Ó7Ó8ˆ
Üœ4 × 1Ñ 1× 8Ñ 8Ó9Ó:ˆä˜*¤iÔ0¼
Øœô9
ð à16×1BÑ1B×1IÑ1IÖJ¨#˜Ÿ™ #™ÐJˆÐJØ�?‰? ,Ñ.Øà" :§?¡?Ñ3ˆä˜+¤yÔ1Øð "×+Ñ+Ó-ˆð ×Ñ ×!2Ñ!2Ò2Ø× Ñ  K×$4Ñ$4Ò4àà�?‰?˜k×.Ñ.Ò.´#°j×6JÑ6JÓ2KÈqÒ2PØô ˆu�{‰{×)Ñ)Ó*¨aÒ/Øð �;‰;× Ò Øð ˜Ÿ™×2Ñ2Ñ2Ø˜EŸK™K×6Ñ6Ñ6ð	
ð7ô
 &Ñ%T°u·{±{×7QÑ7QÓ7SÔ%TÓTÐÜÐÓ  AÒ%ØäœÐ.Ó/Ó0ˆ	ð ˜Ÿ™×2Ñ2°8Ñ<Ò<Ø&ˆOØ'Ñà §¡× :Ñ :¸8Ñ DÒDÐDÐDØ&ˆOØ'ÐåQà—[‘[×%Ñ% aÑ(ˆ
Üˆz‹?˜aÒØàÐÜ—I‘I×'Ñ'¨	Ó2ò 	ˆDØ×#Ñ#Ü—‘× Ñ ×?Ñ?ÀÓEõð	ô  #Ð#3Ó4Ðá2Ð3GÈÐTUÉÓWˆð Ð"Øð
 7<·k±k×6PÑ6PØñ7
ˆ�‰×"Ñ" ?Ñ3ð 8Gˆ�‰×"Ñ"Ð#3Ñ4ð 	×"Ñ" 4¨Ô/Ø×(Ñ(¨°Ó6ˆÜ˜%¤Ô%Ð%Ð%ä�‰Ð;¸UÔCØˆùòm Ks   Å"Qc                ót  — t         j                  rt        d„ ||fD «       «      ry|j                  «       s|j                  «       ry|j                  j                  «       }|j                  j                  «       }||z  }|sy|j                  j                  «       D �ci c]  }|j                  |“Œ }}|j                  j                  «       D �ci c]  }|j                  |“Œ }}g }	|D ]y  }
||
   }||
   }|j                  «       |j                  «       k(  sŒ/|	j                  t        j                  j                  j                  |j                  «       d¬«      ||f«       Œ{ t        |	«      dk(  ryt!        |	t#        j$                  d«      ¬«      \  }}}t'        |t(        «      rt'        |t(        «      sy|j*                  |j*                  k7  r3|j-                  «       |j-                  «       k(  r| j/                  |«      S yd}|j1                  «       s|j3                  ||«      }nV|j1                  «       s|j3                  ||«      }n3t4        j7                  d|j9                  «       |j9                  «       «       |r*t;        j<                  t>        | jA                  ||«      «      S dS c c}w c c}w )a«  
        Right now just greedily reorder the loop of node1 to be compatible with node2,
        but ideally we should have some heuristics to reorder the loop for node2
        to be compatible with node1 if that's more efficient.

        Return the amount of shared data re-computed in this method.
        If no such recomputation happens, return -1 (not return 0 since 0 is a valid
        amount of shared data).

        c              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   rŸ  r  s     rl   r”   z>Scheduler.shared_data_after_reordering_loop.<locals>.<genexpr>®  s   è ø€ ò 8
ØˆA�H‰H�Jñ8
ùrý  r÷   r   rÉ   r$  Fz?Don't reorder loops since both nodes are reductions: %s v.s. %s)!r(   re  ræ   r   r°   r¥  r¶  r²   r’  rÀ   rZ   rœ   r�   rË   rÌ   r©   r¶  r  r  rŒ   r4   r–  r`  Údep_size_hintrŽ   r¯  r™  r  r  rG  rH  r  rI  )rk   r�   r‚   r®  r¯  r°  r¼   r•  r–  Ú
candidatesrü   r—  r˜  Ú_numelÚ	reordereds                  rl   Ú!shared_data_after_reordering_loopz+Scheduler.shared_data_after_reordering_loopž  s‘  € ô  ×0Ò0´Cñ 8
Ø!&¨ ô8
ô 5
ð ð
 ×ÑÔ %×"3Ñ"3Ô"5Øà"×.Ñ.×;Ñ;Ó=ÐØ"×.Ñ.×;Ñ;Ó=Ðà0Ð3EÑEÐÙ"Øà38×3DÑ3D×3UÑ3UÓ3WÖX¨C˜#Ÿ(™( C™-ÐXˆÐXØ38×3DÑ3D×3UÑ3UÓ3WÖX¨C˜#Ÿ(™( C™-ÐXˆÐXð ˆ
Ø.ò 	ˆKØ$ [Ñ1ˆGØ$ [Ñ1ˆGà×3Ñ3Ó5Ø×6Ñ6Ó8ó9ð ×!Ñ!äŸ™×(Ñ(×:Ñ:Ø#×-Ñ-Ó/¸!ð ;ó ð  Øðõð	ô" ˆz‹?˜aÒØô $' z´x×7JÑ7JÈ1Ó7MÔ#NÑ ˆ�˜ä˜'¤9Ô-´ZÀÌÔ5SØà×Ñ˜w×/Ñ/Ò/ð
 × Ñ Ó" g×&7Ñ&7Ó&9Ò9Ø×)Ñ)¨'Ó2Ð2Øàˆ	à×!Ñ!Ô#Ø×7Ñ7¸ÀÓI‰IØ×#Ñ#Ô%Ø×7Ñ7¸ÀÓI‰Iä×#Ñ#ØQØ—‘Ó Ø—‘Ó ôñ ô �K‰Kœ˜T×5Ñ5°e¸UÓCÓDð	
ð ð	
ùòk YùÚXs   ÂJ0ÃJ5c                óf  — t        |t        «      r)|j                  «        xr t        |j                  «       S t        |t
        «      rht        |j                  t        j                  «      r|j                  j                  «        S |j                  «        xr t        |j                  «       S y)z>
        Is this node unfusable under any conditions.
        F)	rŒ   rA  r   rW   r�   r�  r+   r@  rA  r%  s     rl   rO  zScheduler.unfusable_nodeû  s¥   € ô �dÔ2Ô3Ø×'Ñ'Ó)Ð)ò Ô2UØ—	‘	ó3ð /ð ô �dÔ5Ô6Ü˜$Ÿ)™)¤R×%?Ñ%?Ô@ØŸ9™9×6Ñ6Ó8Ð8Ð8Ø×'Ñ'Ó)Ð)ò Ô2UØ—	‘	ó3ð /ð ð rn   c                ó  — |j                  «       t        j                  j                  k  ry|j	                  «       }|j                  «       }d}|||z  kD  r	 |d«       yt        d„ |j                  «       D «       «      }|t        j                  j                  j                  j                  fk(  r	 |d«       yd	d„}|j                  «       }	|	j                  «       s+ ||	j                  «      r|j!                  «       s	 |d«       yy)
zT
        Heuristics to avoid benchmarking predictably slow prologue fusions
        Tçš™™™™™ñ?z@prologue fusion will not increase amount of bytes read in kernelFc              3  ó¨   K  — | ]J  }|j                   �<|j                   j                  «       D ]  }|j                  dk(  r|j                  –— Œ! ŒL y ­w)NÚcall_function)r�   rW  r  rX  )r’   r  r  s      rl   r”   zEScheduler.check_prologue_fusion_heuristics_fusable.<locals>.<genexpr>+  sT   è ø€ ò 
àØ�v‰vÐ!Ø—V‘V×'Ñ'Ó)ò	
ð Ø�t‰t�Ò&ð	 �H�Hð
Øñ
ùs   ‚AAz\prologue fusion will not increase attempt to fuse in padding bc it increases unaligned readsc                ó<   — | j                   dk  xr | j                  S )Nr¨   )ÚitemsizeÚis_floating_point)r¼  s    rl   Úlow_prec_fpzGScheduler.check_prologue_fusion_heuristics_fusable.<locals>.low_prec_fp8  s   € Ø—>‘> QÑ&ÒB¨5×+BÑ+BÐBrn   zVprologue fusion that must be upcast to fp32 not profitable for low precision templates)r¼  ztorch.dtyper†   rv   )rà   rZ   rœ   Úinvoke_quant_opsrl  rn  rª   r–   rŸ  rü  rý  Úconstant_pad_ndr*  rË  rV   r¼  r	  )
rk   Úprologue_noderÕ  r  Ú
read_bytesÚwrite_bytesÚBYTES_THRESHOLD_MULTIPLIERr_  rÏ  Útemplate_bufs
             rl   Ú(check_prologue_fusion_heuristics_fusablez2Scheduler.check_prologue_fusion_heuristics_fusable  s  € ð ×,Ñ,Ó.´!·'±'×2JÑ2JÒJØà"×8Ñ8Ó:ˆ
Ø#×:Ñ:Ó<ˆð &)Ð"Ø˜Ð'AÑAÒBÙÐRÔSØô ñ 
à"×,Ñ,Ó.ô
ó 
ˆð ”u—y‘y—~‘~×5Ñ5×=Ñ=Ð?Ò?ÙØnôð ó	Cð %×?Ñ?ÓAˆà×6Ñ6Ô8Ù˜L×.Ñ.Ô/Ø!×>Ñ>Ô@áØhôð àrn   c                ó<  ‡ — t        |t        «      rt        |t        «      syt        |j                  t        j                  «      r$t        |j                  t        j                  «      sy|j                  «       s|j                  «       ryt        j                  dk(  ry|j                  |j                  }}|\  }}|\  }}|j                  «       s,|j                  «       s||k7  st        |«      t        |«      k7  ryt        |j                  j                  «      dkD  s"t        |j                  j                  «      dkD  ry‰ j                  t        t        |j                  j                  «      «      «      }	‰ j                  t        t        |j                  j                  «      «      «      }
t!        |	|
«      t        j"                  kD  rydˆ fd„} ||«      s ||«      ryg }t%        t'        ||«      «      D ]  \  }\  }}||k7  sŒ|j)                  |«       Œ! t        |«      dk7  ry|d   }||   ||   }}t*        j,                  j.                  j1                  ||«      r|||fS t*        j,                  j.                  j1                  ||«      r|||fS y)ao  
        Fusing two small pointwise nodes significantly reduces kernel overhead
        and launch overhead. However, slightly different sizes would prevent fusion.
        Here, we decide if expanding sizes of one node is profitible by allowing
        fusion, and returns the dimension to expand, node with smaller sizes,
        and new size after expand.
        Nri  r   c                ó~  •— | j                   j                  D ]£  }|j                  ‰j                  v r‰j                  |j                     }n%‰j                  j                  |j                  «      }|sŒ]t        j                  j                  j                  || «      sŒˆt        |j                  t        «      rŒ£ y yr  )r°   r±   r²   r?  r@  rÈ  rZ   rœ   r:  r@  rŒ   r  rA  )r�   rH  rI  rk   s      €rl   Úhas_reusable_bufferzIScheduler.get_expand_dim_for_pointwise_nodes.<locals>.has_reusable_buffer‚  s–   ø€ Ø×(Ñ(×.Ñ.ò  �à—9‘9 × ;Ñ ;Ñ;Ø $× ;Ñ ;¸D¿I¹IÑ F‘Ià $× 0Ñ 0× 4Ñ 4°T·Y±YÓ ?�Iò ÜŸ™×,Ñ,×6Ñ6°yÀ$ÕGÜ& y×'<Ñ'<Ô>TÕUáð ð rn   r   r	  )rŒ   r�   r�   r+   r   rÂ  r(   rj  rY  rŽ   r©   r°   r�  rÂ  rÐ  rÈ  r¶  Úsmall_memory_access_thresholdrÑ  r±  rÀ   rZ   rœ   r�   Ústatically_known_lt)rk   r�   r‚   Ún1_sizesÚn2_sizesÚn1_iter_sizesÚn1_reduce_sizesÚn2_iter_sizesÚn2_reduce_sizesÚnode1_write_memoryÚnode2_write_memoryrÚ  Úmismatch_dimensionsÚidxÚn1_sizeÚn2_sizeÚmismatch_dimÚmismatch_size1Úmismatch_size2s   `                  rl   Ú"get_expand_dim_for_pointwise_nodesz,Scheduler.get_expand_dim_for_pointwise_nodesH  s]  ø€ ô ˜%¤Ô/´zÀ%ÌÔ7WØô �u—z‘z¤2×#4Ñ#4Ô5Ü˜5Ÿ:™:¤r×'8Ñ'8Ô9àð ×)Ñ)Ô+¨u×/MÑ/MÔ/OØô ×Ñ Ò)Øð #Ÿ\™\¨5¯<©<�(ˆØ)1Ñ&ˆ�Ø)1Ñ&ˆ�à×ÑÔ Ø×!Ñ!Ô#Ø /Ò1Ü�=Ó!¤S¨Ó%7Ò7àô ˆu× Ñ ×'Ñ'Ó(¨1Ò,´°E×4EÑ4E×4LÑ4LÓ0MÐPQÒ0QØð "×/Ñ/´´T¸%×:KÑ:K×:RÑ:RÓ5SÓ0TÓUÐØ!×/Ñ/´´T¸%×:KÑ:K×:RÑ:RÓ5SÓ0TÓUÐäÐ"Ð$6Ó7Ü×2Ñ2ò3ð õ	ñ  ˜uÔ%Ñ)<¸UÔ)CØð !ÐÜ'0´°]ÀMÓ1RÓ'Sò 	0Ñ#ˆCÑ#�'˜7Ø˜'Ó!Ø#×*Ñ*¨3Õ/ð	0ô Ð"Ó# qÒ(Øà*¨1Ñ-ˆà˜,Ñ'Ø˜,Ñ'ð 'ˆô �7‰7×Ñ×/Ñ/°ÀÔOØ ¨Ð6Ð6Ü�W‰W×Ñ×1Ñ1°.À.ÔQØ ¨Ð6Ð6àrn   c                óT  ‡‡‡‡‡— ‰‰u ry| j                  «       r@| j                  j                  ‰«      }| j                  j                  ‰«      }|�|�||k7  ryt        ‰t        «      r‰j                  ‰«      S t        ‰t        «      ryt        ‰‰«      }‰j                  «       r0| j                  ‰j                  «       «      j                  ‰‰«      ryt        ‰t        «      st        ‰t        «      r	 |d«       yt        ‰t        «      r‰j                  «       s	 |d«       yt        ‰t        «      �r%t        ‰j                  t        j                   «      s	 |d«       y‰j                  j#                  «       s	 |d«       yt        ‰t$        «      s	 |d«       yt        ‰j                  t&        «      s	 |d	«       yt        ‰j                  j(                  t*        «      s	 |d
«       yt-        ‰j                  j.                  «      dk(  sJ ‚‰j                  j.                  d   j0                  Št3        ˆfd„‰j4                  j6                  D «       «      r	 |d«       y‰j                  j(                  j9                  «       }|D ];  }	‰j                  j(                  j;                  |	«      }
t3        d„ |
D «       «      sŒ; y t-        ‰j                  j<                  «      dk(  sJ ‚‰j                  j<                  d   j>                  ‰j                  j>                  k7  r	 |d«       y	 	 d%ˆˆˆfd„Št3        ˆfd„| j@                  D «       «      ryt        ‰t        t        f«      r‰j                  «       s	 |d«       y‰jC                  «       ‰jD                  z  r	 |d«       y‰j                  «       �rtG        ‰«      s	 |d«       y‰jI                  «       s‰j                  «       r	 |d«       y‰jK                  «       }|jM                  «       }|s	 |d«       ytO        d„ |jP                  D «       «      |z
  }‰jS                  «       |z  r	 |d«       y‰jU                  «       s‰jU                  «       r	 |d«       y‰jW                  «       Š‰dd D ]B  }|jY                  «       }|D ]+  }t[        ˆfd„|j\                  D «       «      rŒ" |d«         y ŒD t        ‰t^        «      s‰gn*‰j`                  D �cg c]  }|j                  «       sŒ|‘Œ c}}t-        |«      dk(  sJ ‚|d   }t-        ‰d   jb                  «      dk(  rSt-        ‰d   jb                  d   j\                  «      dk(  r+‰d   jb                  d   j\                  d   j                  |u s	 |d«       y| je                  ‰‰|«      sy‰j                  «       r…‰jU                  «       s‰jI                  «       stg        ‰«      s	 |d«       y‰ji                  «       }|€J ‚|jk                  «       r-t        ‰j                  t        j&                  «      s	 |d «       y‰jS                  «       tl        jn                  jp                  z  s+‰jS                  «       tl        jn                  jp                  z  r	 |d!«       y‰j                  «       }‰j                  «       }||k7  r |d"||«       y~| js                  ‰‰|¬#«      }t        |tt        «      sJ ‚|r<|tv        jx                  k  r)tv        jz                  r| j}                  ‰‰«      }|dk\  r|}tv        j~                  rP| j�                  ‰‰«      x}r<|\  }}}|jƒ                  ||«       | js                  ‰‰«      }t        |tt        «      sJ ‚tv        j„                  r,|tv        jx                  k  r| j‡                  ‰‰«      }|dk\  r|}tˆ        j‹                  tŒ        jŽ                  «      r4tˆ        j‘                  d$‰j“                  «       ‰j“                  «       |«       tl        j”                  j—                  | ‰‰|«      sy‰jC                  «       ‰jD                  z  rY| j™                  ‰‰«      xrE tl        j”                  j™                  | ‰‰|«      xr! | j                  |«      j™                  ‰‰«      S tl        j”                  j›                  | ‰‰|«      xr! | j                  |«      j›                  ‰‰«      S c c}w )&zj
        Determine if it is possible to combine node1 and node2 into a
        single fused node.
        FNTz/grouped node must not be fused with other nodesznode1 is nopz'node1 is extern but not a triton kernelz5node1's triton kernel doesn't support epilogue fusionz.node1 is extern but node2 is not SchedulerNodez3node1 is extern but node2.node is not SchedulerNodez4node1 is extern but node2.node.data is not Pointwiser   r   c              3  ó<   •K  — | ]  }|j                   ‰k7  –— Œ y ­wrw   r³  )r’   r¼   Úwritten_buffer_names     €rl   r”   z%Scheduler.can_fuse.<locals>.<genexpr>î  s   øè ø€ ÒV°s�3—8‘8Ð2Õ2ÑVùrT  z9epilogue reads from buffers other than the mutated outputc              3  ó&   K  — | ]	  }|d k7  –— Œ y­w)r·  Nr~   )r’   Úusages     rl   r”   z%Scheduler.can_fuse.<locals>.<genexpr>÷  s   è ø€ Ò;¨5�u •Ñ;ùs   ‚z*node1 and node2 uses different buf layoutsc                ó@   •— | ‰uxr | ‰uxr ‰| j                  «       v S rw   )r¿   )rê   r�   r‚   rï  s    €€€rl   Ú._is_other_node_that_references_mutation_bufferzJScheduler.can_fuse.<locals>._is_other_node_that_references_mutation_buffer   s7   ø€ ð   uÐ,ò NØ#¨5Ð0òNà+¨z×/KÑ/KÓ/MÐMðrn   c              3  ó.   •K  — | ]  } ‰|«      –— Œ y ­wrw   r~   )r’   r�   ró  s     €rl   r”   z%Scheduler.can_fuse.<locals>.<genexpr>	  s   øè ø€ ò àñ ?¸t×Dñùr>  znode2 is extern or nopznode1 must go before node2zprologue fusion turned offz2prologue fusion only supported for pointwise nodesz'template has no allowed prologue inputsc              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   rø  )r’   r‡  s     rl   r”   z%Scheduler.can_fuse.<locals>.<genexpr>*  s   è ø€ ÒE¨c˜3Ÿ<™<Ÿ>ÑEùrý  z;prologue fusion not implemented for kernel for these inputsz:template prologue can only fuse functional pointwise nodesr÷   c              3  ó:   •K  — | ]  }|j                   ‰v –— Œ y ­wrw   r—   )r’   r.  Úprologue_nodess     €rl   r”   z%Scheduler.can_fuse.<locals>.<genexpr>:  s   øè ø€ ÒQ¸t˜tŸy™y¨NÔ:ÑQùrÄ  z7template prologue can only fuse nodes with a single usezEtemplate prologue can only fuse nodes with a single use into templateztemplate epilogue not satisfiedz6multi-output template epilogue requires ComputedBufferz#fusion for buffer explicit disabledzdevice mismatch (%s vs %s)r;  z%s and %s has %s shared data)rê   r_   )NrÂ  r¡  rÈ  rŒ   r8  rO  r  r   rc  rÞ   Úcan_fuse_multi_outputs_templaterì  rA  r�  r�   r+   r@  rA  r�   r   rÔ   r   r©   Úmutation_outputsr²   ræ   r°   r±   Úinner_fn_free_symbolsÚcollect_inner_fn_symbol_usagerC  r'  rÒ  rà   rß   rB  rŽ   rË  Úget_allowed_prologue_inpsr   ró  r  rÂ  r–   rz  r•   r  rµ   r·   rj  r×  r=  rÆ  rV   rZ   rœ   Úno_fuse_buffer_namesrI  r  r(   Úscore_fusion_memory_thresholdre  rÆ  Ú$expand_dimension_for_pointwise_nodesrì  r�  rn  r¿  r™  rÓ  rÔ  rÆ  r  r  r  rî   Úcan_fuse_verticalÚcan_fuse_horizontal) rk   r�   r‚   Úcan_reorderr<  r  Ústream2r  Únode2_inner_fn_free_symbolsÚsymbolÚusagesrÊ  rÐ  Úunsupported_prologue_argsr�   Ú	node_outsrÁ   r  Útemplate_snodesÚtemplate_snoderÖ  r  Údevice2Úshared_data_scoreÚnew_shared_data_scoreÚexpand_analysisÚ
expand_dimÚsmaller_nodeÚexpand_sizeró  r÷  rï  s     ``                          @@@rl   rî   zScheduler.can_fuseª  sR  ü€ ð �E‰>Øð ×'Ñ'Ô)Ø×)Ñ)×-Ñ-¨eÓ4ˆGØ×)Ñ)×-Ñ-¨eÓ4ˆGØÐ" wÐ':¸wÈ'Ò?QØä�eÔ4Ô5Ø×&Ñ& uÓ-Ð-Ü�eÔ4Ô5ð ä˜˜uÓ%ˆà×ÑÔ 4×#3Ñ#3Ø×ÑÓó$
ç
)Ñ
)¨%°Ó
7ð$8ð ä�eÔ1Ô2´jØÔ'ô7
ñ ÐAÔBØÜ�eÔ3Ô4¸U×=NÑ=NÔ=PÙ�ÔØä�eÔ6Õ7Ü˜eŸj™j¬"×*DÑ*DÔEÙÐ=Ô>Øà—:‘:×/Ñ/Ô1ÙÐKÔLØä˜e¤]Ô3ÙÐDÔEØÜ˜eŸj™j¬.Ô9ÙÐIÔJØÜ˜eŸj™jŸo™o¬yÔ9ÙÐJÔKØä�u—z‘z×2Ñ2Ó3°qÒ8Ð8Ð8Ø"'§*¡*×"=Ñ"=¸aÑ"@×"EÑ"EÐô ÓV¸e×>OÑ>O×>UÑ>UÔVÔVÙÐOÔPØð +0¯*©*¯/©/×*OÑ*OÓ*QÐ'Ø5ò !�ØŸ™Ÿ™×FÑFÀvÓN�ÜÑ;°FÔ;Õ;Ù ð!ô �u—z‘z×.Ñ.Ó/°1Ò4Ð4Ð4Ø�z‰z×&Ñ& qÑ)×0Ñ0°E·J±J×4EÑ4EÒEÙÐ@ÔAØðØ-÷ô ó à ŸJ™Jôô ð ô �uÔ8Ô:PÐQÔRØ×%Ñ%Ô'áÐ(Ô)Øà×$Ñ$Ó&¨¯©Ò8ÙÐ,Ô-Øà×ÑÕÜ.¨uÔ5ÙÐ0Ô1Øà×!Ñ!Ô# u×'8Ñ'8Ô':ÙÐHÔIØà×7Ñ7Ó9ˆHØ$,×$FÑ$FÓ$HÐ!Ù(ÙÐ=Ô>Øô ÑE°X·_±_ÔEÓEØ'ñ(ð &ð
 ×%Ñ%Ó'Ð*CÒCÙÐQÔRØà×-Ñ-Ô/°5×3QÑ3QÔ3SÙÐPÔQØà"Ÿ_™_Ó.ˆNØ& s¨Ð+ò %�Ø ×,Ñ,Ó.�	Ø$ò %�CÜÓQÀsÇyÁyÔQÕQÙÐUÔVÚ$ñ%ð%ô " %Ô);Ô<ð ‘à!&§¡ÖA˜A°·±µ’aÒAð ô
 �Ó'¨1Ò,Ð,Ð,Ø,¨QÑ/ˆNô �N 2Ñ&×.Ñ.Ó/°1Ò4Ü˜ rÑ*×2Ñ2°1Ñ5×;Ñ;Ó<ÀÒAØ" 2Ñ&×.Ñ.¨qÑ1×7Ñ7¸Ñ:×?Ñ?À>ÑQáØ[ôð à×@Ñ@ÀÈÈsÔSØà×ÑÔà×.Ñ.Ô0Ø×%Ñ%Ô'Ü2°5Ô9áÐ5Ô6ØØ ×2Ñ2Ó4ˆLØÐ+Ð+Ð+Ø×5Ñ5Ô7Ä
Ø—
‘
œB×-Ñ-ôAñ ÐLÔMØà×"Ñ"Ó$¤q§w¡w×'CÑ'CÒCØ×"Ñ"Ó$¤q§w¡w×'CÑ'CÒCáÐ5Ô6ØØ×!Ñ!Ó#ˆØ×"Ñ"Ó$ˆØ�WÒÙÐ,¨f°gÔ>ØØà ×4Ñ4Ø�5Ð4Mð 5ó 
Ðô Ð+¬SÔ1Ð1Ð1ñ Ø!¤F×$HÑ$HÒHÜ×1Ò1à$(×$JÑ$JÈ5ÐRWÓ$XÐ!Ø$¨Ò)Ø$9Ð!ä×6Ò6Ø#×FÑFÀuÈeÓTÐTˆOÐTà6EÑ3ˆZ˜ {Ø×<Ñ<¸ZÈÔUØ $× 8Ñ 8¸ÀÓ FÐÜÐ/´Ô5Ð5Ð5ô ×1Ò1Ø!¤F×$HÑ$HÒHà$(×$MÑ$MØ�uó%Ð!ð %¨Ò)Ø$9Ð!ä×)Ñ)¬'¯-©-Ô8Ü×#Ñ#Ø.Ø—‘Ó Ø—‘Ó Ø!ô	ô �y‰y×!Ñ! $¨¨uÐ6GÔHØà×$Ñ$Ó&¨¯©Ò8ð ×&Ñ& u¨eÓ4ò MÜ—I‘I×/Ñ/°°e¸UÐDUÓVòMà×$Ñ$ VÓ,×>Ñ>¸uÀeÓLðô —9‘9×0Ñ0Ø�e˜UÐ$5óò Mà×"Ñ" 6Ó*×>Ñ>¸uÀeÓLðMùòC Bs   Ôd%Ô$d%c                óˆ  — |j                  «       }t        ||«      }t        t        «      }|j                  D ]j  }| j
                  j                  |j                  |j                  «      }t        |t        «      r| j                  |||«      rŒW||   j                  |«       Œl |j                  j                  D ]à  }t        |t        «      st        |t        «      sŒ$|j                  | j
                  j                  |j                  |j                  «      «      }	|	sŒf|	D ]v  }
t        |t        «      r$| j!                  |
|«      r|	j#                  |
«       Œ7t        |t        «      sŒH| j%                  |
||j&                  «      sŒf|	j#                  |
«       Œx Œâ t)        d„ t*        j,                  j/                  |j1                  «       «      D «       «      }||z  r	 |d«       y|j3                  «       }|D ]E  }| j4                  |   j7                  «       }|| j8                  |   j:                  z  sŒ= |d«        y y)a  
        Check if it is legal to fuse a consumer (node2) into a producer (node1).

        We can fuse them if all the reads of node2 either match
        corresponding writes in node1, or are written by nodes that can
        be scheduled before the fusion of node1 and node2.
        c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   z.Scheduler.can_fuse_vertical.<locals>.<genexpr>Í  s   è ø€ ò $
àð �H�Hñ$
ùrµ  zmemory deps did not matchFz(intermediate nodes between node1 & node2T)r  r  r   rÿ   rq  rg  rÈ  r²   rŒ   r6   r5  rÀ   r°   r�  r4   r5   Úfusable_read_and_writer\  Ú.fusable_stardep_write_and_read_on_empty_tensorr�   r   rÔ  rÕ  r²  r¹   rà   r@  r  ró  rß   )rk   r�   r‚   Únode1_buf_namesr  Úremaining_deps_by_namer¼   r²   ÚcdÚ	remainingrx  Úremaining_depsÚnode1_op_namesrè  s                 rl   r   zScheduler.can_fuse_vertical¦  s  € ð  ×0Ñ0Ó2ˆÜ˜˜uÓ%ˆÜ7BÄ4Ó7HÐà×+Ñ+ò 	5ˆCØ×(Ñ(×,Ñ,¨S¯X©X°s·x±xÓ@ˆDÜ˜#œwÔ'¨D×,AÑ,AÀ#ÀuÈeÔ,TØØ" 4Ñ(×/Ñ/°Õ4ð		5ð ×#Ñ#×*Ñ*ò 	-ˆBÜ˜b¤)Ô,´ZÀÄGÔ5LØØ.×2Ñ2Ø×%Ñ%×)Ñ)¨"¯'©'°2·7±7Ó;óˆIò Ø#ò 
-�BÜ! "¤iÔ0°T×5PÑ5PØ˜Bô6ð "×(Ñ(¨Õ,Ü#ØœGõà×MÑMØ˜B §
¡
õð "×(Ñ(¨Õ,ñ
-ð	-ô& $ñ $
ä —‘×4Ñ4Ð5K×5RÑ5RÓ5TÓUô$
ó 
ˆð
 ˜OÒ+ñ
 Ð+Ô,Øà×2Ñ2Ó4ˆØ"ò 	ˆDØ×&Ñ& tÑ,×=Ñ=Ó?ˆGØ × 7Ñ 7¸Ñ @× JÑ JÓJÙÐ>Ô?Ùð		ð rn   c                ó  ‡— |j                   |j                  «       vry|j                  j                  D �cg c]  }|j                   |j                  k(  r|‘Œ }}t        |«      dk7  ry|d   Št        ‰t        «      ryt        ‰t        «      sJ ‚t        ‰j                  t        j                  «      ryt        ‰j                  «      ‰j                  j                  k  sy| j                   |j                     }|g}t        |t"        «      r|j$                  }d}|D ]R  }	|	j                  j&                  D �
cg c]  }
|
j                   |k(  r|
‘Œ }}
|sŒ8|dz  }t)        ˆfd„|D «       «      rŒR y |dk  S c c}w c c}
w )NFr   r   c              3  óø   •K  — | ]q  }t        |t        «      xr[ t        |j                  t        j
                  «       xr4 |j                  ‰j                  k(  xr |j                  ‰j                  k(  –— Œs y ­wrw   )rŒ   r4   r#   r³   r%   ÚTMPr¸   )r’   rH  Úwrites     €rl   r”   z-Scheduler.fusable_weak_dep.<locals>.<genexpr>  sm   øè ø€ ò ð
 ô	 ˜4¤Ó+ò ,Ü+¨D¯J©J¼¿¹ÓAÐAò,à—J‘J %§+¡+Ñ-ò,ð —I‘I §¡Ñ+ó,ñùs   ƒA7A:)r²   r  r°   r�  rù  r©   rŒ   r5   r4   r#   r³   r%   r  r   r¦  r"   rÊ  ro  r·   r±   r•   )rk   Úweak_depr�   r‚   r  Úmutating_writesr«  Úrelevant_reading_nodesÚnum_concurrent_readsÚreading_noderH  Úrelevant_readss       `       rl   r5  zScheduler.fusable_weak_depã  sŽ  ø€ ð �=‰= × 6Ñ 6Ó 8Ñ8Øð ×*Ñ*×1Ñ1ö
àØ�z‰z˜X×2Ñ2Ò2ò ð
ˆð 
ô
 ˆÓ 1Ò$ØØ Ñ"ˆÜ�eœWÔ%ØÜ˜%¤Ô+Ð+Ð+ä˜uŸ{™{¬D¯H©HÔ5Øô
 ˜%Ÿ/™/Ó*¨e¯k©k×.FÑ.FÒFØà×+Ñ+¨H×,AÑ,AÑBˆ	Ø"' ÐÜ�eÔ7Ô8Ø%*§\¡\Ð"Ø ÐØ2ò 	ˆLð )×4Ñ4×:Ñ:öàØ—9‘9 	Ò)ò ðˆNð ñ
 "ØØ  AÑ%Ð Üó ð
 +ôõ ñ ð!	ð" $ qÑ(Ð(ùòW
ùò6s   ·"E:Ä;E?c                ó$  — t        |t        «      �rd| j                  j                  |j                  |j                  «      }||j                  k7  sHt        |j                  t        j                  «      s$t        |j                  t        j                  «      ryt        j                  r9|j                  |j                  k7  r |j                  «       }|j                  «       }| j                  |j                  «      ry|j                  |j                  k(  xr\ t        |j                   «      t        |j                   «      k\  xr/ |j                   d t        |j                   «       |j                   k(  S t        |t"        «      r‹| j                  j                  |j                  |j                  «      }| j                  j                  |j                  |j                  «      }|j                  |j                  k(  r|j                  �||k(  ryyrF  )rŒ   r4   rg  rÈ  r²   r#   r³   r%   r  r(   re  r–  r`  rD  rÍ  r©   r¸   r5   )rk   rH  r  Ú	read_nameÚ
write_names        rl   r  z Scheduler.fusable_read_and_write  sw  € Ü�dœIÕ&Ø×-Ñ-×1Ñ1°$·)±)¸T¿Y¹YÓGˆIð ˜UŸZ™ZÒ'Ü& t§z¡z´4·8±8Ô<Ü& u§{¡{´D·H±HÔ=àä×0Ò0°T·]±]ÀeÇnÁnÒ5Tð —~‘~Ó'�ØŸ™Ó)�ð ×1Ñ1°%·*±*Ô=Øð —
‘
˜eŸk™kÑ)ò ?Ü˜Ÿ	™	“N¤c¨%¯*©*£oÑ5ò?à—I‘IÐ/¤ E§J¡J£Ð0°E·J±JÑ>ðô
 ˜œgÔ&Ø×-Ñ-×1Ñ1°$·)±)¸T¿Y¹YÓGˆIØ×.Ñ.×2Ñ2°5·:±:¸u¿z¹zÓJˆJà—	‘	˜UŸZ™ZÒ'Ø—J‘JÐ*Ø Ò+àØrn   c                óH  — t        |t        j                  «      sy|j                  «       sy| j                  j                  |j                  |j                  «      }| j                  j                  |j                  |j                  «      }t        |t        «      r||k(  ryyrF  )rŒ   r+   r@  rA  rg  rÈ  r²   r5   )rk   rH  r  Úwriting_noder'  r(  s         rl   r  z8Scheduler.fusable_stardep_write_and_read_on_empty_tensorF  s}   € ô ˜,¬×(BÑ(BÔCØØ×-Ñ-Ô/ØØ×)Ñ)×-Ñ-¨d¯i©i¸¿¹ÓCˆ	Ø×*Ñ*×.Ñ.¨u¯z©z¸5¿:¹:ÓFˆ
Ü�eœWÔ%¨)°zÒ*AØØrn   c                óB   — t         j                  j                  ||«      S rw   )rZ   rœ   Úget_dep_size_hint)rk   r¼   rE  s      rl   rÂ  zScheduler.dep_size_hintS  s   € Ü�w‰w×(Ñ(¨¨kÓ:Ð:rn   c           	     óB  ‡ ‡‡— ˆfd„}|r6t         j                  ||«      r t         j                  ||«      } ||dd«      S t        |j                  t
        j                  «      r|j                  j                  «       s |j                  «       s|j                  «       r¯|j                  j                  |j                  j                  z  }|j                  j                  |j                  j                  z  }	d	d„}
d}|D ]@  }|	D ]9  } |
||«      sŒ|t        ‰ j                  |«      ‰ j                  |«      «      z  }Œ; ŒB  ||dd«      S t        |j                  j                  «      t        |j                  j                  «      z   }t        |j                  j                  «      t        |j                  j                  «      z   }t        ||«      dz  t        ||«      k  r�||kD  r||}}|j                  j                  |j                  j                  z  D �cg c]4  }||j                  j                  v s||j                  j                  v r|‘Œ6 }} |t!        ˆˆ fd„|D «       «      dd«      S |j                  j                  |j                  j                  z  |j                  j                  |j                  j                  z  z  }t!        ˆ fd„|D «       «      }d}|dk(  r$‰ j#                  ||«      r‰ j%                  ||«      } |||d«      S c c}w )
a4  
        The first term in our fusion score that estimates number of saved
        memory operations.

        This function scores fusion candidates based on shared memory access patterns.
        Higher scores indicate better fusion candidates.

        Scoring strategy:
        1. If nodes share exact memory deps (same buffer + same indexing), return
           the sum of shared dep sizes (original behavior).
        2. If no exact matches (score == 0), check for same-buffer reads with
           different indexing (e.g., split operations reading different slices).
           - Give bonus if nodes read from exactly the same set of buffers
           - Score based on overlap ratio: common_buffer_size / total_read_size
           - High overlap (>50%) suggests good cache locality benefit from fusion
        c                ó   •— ‰r| ||fS | |z   S rw   r~   )r¾  Úbuffer_overlap_scoreÚis_mix_order_reductionÚreturn_is_mix_order_reductions      €rl   Ú_construct_return_valuez>Scheduler.score_fusion_memory.<locals>._construct_return_valueo  s#   ø€ ñ -ØÐ3Ð5KÐLÐLØÐ/Ñ/Ð/rn   r   Tc                óš   — | |k(  ryt        | t        t        f«      r/t        |t        t        f«      r| j                  |j                  k(  S yr  )rŒ   r5   r4   r²   )Údep1Údep2s     rl   Ú_matchz-Scheduler.score_fusion_memory.<locals>._match�  sD   € Ø˜4’<ØÜ˜d¤W¬iÐ$8Ô9¼jØœ7¤IÐ.ô?ð  Ÿ9™9¨¯	©	Ñ1Ð1Ørn   Fr&  c              3  óB   •K  — | ]  }‰j                  |‰«      –— Œ y ­wrw   ©rÂ  )r’   r¼   rE  rk   s     €€rl   r”   z0Scheduler.score_fusion_memory.<locals>.<genexpr>±  s   øè ø€ ÒI¸S�D×&Ñ& s¨K×8ÑIùrË  c              3  ó@   •K  — | ]  }‰j                  |«      –— Œ y ­wrw   r8  rã  s     €rl   r”   z0Scheduler.score_fusion_memory.<locals>.<genexpr>¸  s   øè ø€ ÒJ°�D×&Ñ& s×+ÑJùr1  )r4  r3   r5  r3   )r‰   rî   rÎ   rŒ   r�   r+   r@  rA  r   r°   r±   r�  r¶  rÂ  r©   r‰  rÙ   Ú_can_use_buffer_overlap_scoringÚ&_score_fusion_memory_by_buffer_overlap)rk   r�   r‚   rE  r1  r<  r2  r¾  Ú
node1_depsÚ
node2_depsr6  Ú	node1_depÚ	node2_depÚnode1_dep_lenÚnode2_dep_lenr¼   rß  Úcommon_memory_depsr/  s   `  ``              rl   rI  zScheduler.score_fusion_memoryV  så  ú€ ô2	0ñ %Ô):×)CÑ)CÀEÈ5Ô)Qô
 &×6Ñ6°u¸eÓDˆEÙ*¨5°!°TÓ:Ð:ô ˜5Ÿ:™:¤r×'AÑ'AÔBØ—J‘J×0Ñ0Ô2à× Ñ Ô"Ø× Ñ Ô"à×*Ñ*×0Ñ0°5×3DÑ3D×3KÑ3KÑKˆJØ×*Ñ*×0Ñ0°5×3DÑ3D×3KÑ3KÑKˆJóð ˆEØ'ò �	Ø!+ò �IÙ˜i¨Õ3Ø¤Ø ×.Ñ.¨yÓ9¸4×;MÑ;MÈiÓ;Xó"ñ ™ñðñ +¨5°!°UÓ;Ð;ä˜E×-Ñ-×3Ñ3Ó4´s¸5×;LÑ;L×;SÑ;SÓ7TÑTˆÜ˜E×-Ñ-×3Ñ3Ó4´s¸5×;LÑ;L×;SÑ;SÓ7TÑTˆô ˆ}˜mÓ,¨qÑ0´3°}ÀmÓ3TÒTØ˜}Ò,Ø$ e�u�ð !×,Ñ,×2Ñ2°U×5FÑ5F×5MÑ5MÑMöàØ˜%×+Ñ+×1Ñ1Ñ1°S¸E×<MÑ<M×<TÑ<TÑ5Tò ðˆDð ñ +ÜÔIÀDÔIÓIÈ1Èeóð ð $×/Ñ/×5Ñ5¸×8IÑ8I×8PÑ8PÑPØ×Ñ×#Ñ# e×&7Ñ&7×&>Ñ&>Ñ>ñ
Ðô ÓJÐ7IÔJÓJˆð
  !ÐØ�AŠ:˜$×>Ñ>¸uÀeÔLØ#'×#NÑ#NØ�uó$Ð ñ ' uÐ.BÀEÓJÐJùò3s   È 9Lc                ót  — |j                  «       s|j                  «       ry|j                  «       s|j                  «       ryt        j                  st        j                  �rU|j                  «       }|j                  «       }|r|syt        d„ |D «       «      }t        d„ |D «       «      }t        «       }|D ]â  }|j                  D ]Ñ  }	t        |	j                  t        «      sŒ|	j                  j                  «       sŒ9t        |	j                  «      sŒO|	j                  j                  «       }
|
�Lt        |
t        j                  «      r2|
j                  «       }||z  sŒ›|j!                  |	j                  «       Œ·|j!                  |	j                  «       ŒÓ Œä |r®|D ]©  }|j                  D ]˜  }	t        |	j                  t        «      sŒ|	j                  j                  «       sŒ9|	j                  |v sŒH|	j                  j                  «       }
|
�3t        |
t        j                  «      r|
j                  «       }||z  sŒ”  y  y Œ« ||fD ]d  }|j"                  j$                  D ]I  }| j&                  j)                  |j*                  «      }|€Œ+|j                  «       sŒ<t-        |«      sŒH  y Œf y)a@  
        Check if buffer overlap scoring should be used for this node pair.

        Buffer overlap scoring handles split/cat patterns where nodes read from
        the same buffer at different indices. We skip it when:
        - Either node is a reduction (different memory access patterns)
        - Either node is a template
        - Both nodes are prologue/epilogue candidates for the same template,
          because horizontal fusion would prevent them from being absorbed
          into the template kernel. For example, in:
            q = a[:64, :]; k = a[64:, :]
            return mm(q + 2, k - 2)
          "q + 2" and "k - 2" both read from `a` and would get a high overlap
          score, but fusing them horizontally prevents prologue fusion into mm
          (resulting in 2 kernels instead of 1).

        We allow buffer overlap scoring when:
        - The node outputs are not actually in the template's allowed_prologue_inps,
          meaning they can't be prologue-fused anyway, so horizontal fusion doesn't
          prevent any optimization opportunity.
        FTc              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   rø  rÁ  s     rl   r”   z<Scheduler._can_use_buffer_overlap_scoring.<locals>.<genexpr>ì  ó   è ø€ Ò+T¸s¨C¯L©L¯NÑ+Tùrý  c              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrw   rø  rÁ  s     rl   r”   z<Scheduler._can_use_buffer_overlap_scoring.<locals>.<genexpr>í  rE  rý  )rŽ   r   r(   r�  rŒ  rz  r   r  rŒ   r�   r_   rB  rÆ  r+   r   rü  rÜ  r°   r±   ró  rÈ  r²   r=  )rk   r�   r‚   Únode1_outputsÚnode2_outputsÚnode1_output_namesÚnode2_output_namesÚ&node1_prologue_eligible_template_usersrº   r.  rÕ  Úallowed_inpsr�   r¼   rr  s                  rl   r:  z)Scheduler._can_use_buffer_overlap_scoringÅ  sf  € ð4 ×ÑÔ 5×#5Ñ#5Ô#7ØØ×ÑÔ %×"3Ñ"3Ô"5Øä×Ò¤&×":Ó":Ø!×-Ñ-Ó/ˆMØ!×-Ñ-Ó/ˆMñ !©Øä!+Ñ+TÀmÔ+TÓ!TÐÜ!+Ñ+TÀmÔ+TÓ!TÐô
 “ð 3ð %ò R�ØŸI™Iò R�Dä" 4§9¡9Ô.?Õ@Ø ŸI™I×1Ñ1Õ3Ü7¸¿	¹	ÕBð
 )-¯	©	×(CÑ(CÓ(E˜Ø(Ð4¼Ø)¬2×+BÑ+Bô:ð ,9×+RÑ+RÓ+T˜LØ1°LÓ@Ø F× JÑ JÈ4Ï9É9Õ Uð C×FÑFÀtÇyÁyÕQñ%RðRñ. 6Ø(ò -�CØ #§	¡	ò -˜ä& t§y¡yÔ2CÕDØ $§	¡	× 5Ñ 5Õ 7Ø $§	¡	Ð-SÒ Sð -1¯I©I×,GÑ,GÓ,I˜MØ,Ð8¼ZØ -¬r×/FÑ/Fô>ð 0=×/VÑ/VÓ/X Ø#5¸Ó#DÚ+0ò (-ñ!-ð-ð&  ˜ò %�Ø×+Ñ+×1Ñ1ò %�CØ#×6Ñ6×:Ñ:¸3¿8¹8ÓD�Hà Ñ,Ø$×0Ñ0Õ2Ü7¸ÕAâ$ñ%ð%ð rn   c                ó|  ‡ ‡‡‡— dŠd
ˆˆ fd„Št        d„ |j                  j                  D «       «      }t        d„ |j                  j                  D «       «      }||z  Š‰syt        ˆfd„|j                  j                  D «       «      }t        ˆfd„|j                  j                  D «       «      }t	        ||«      }|dk(  ryt        ˆˆfd„|j                  j                  D «       «      }t        ˆˆfd	„|j                  j                  D «       «      }	t	        ||	«      }
|
|z  }|t
        j                  k\  r|
S dS )a8  
        Score fusion based on buffer name overlap when exact dep matching fails.

        This handles the split/cat fusion case where nodes read from the same buffer
        but at different indices (e.g., different slices from a split operation).

        Scoring logic:
        - If nodes read from exactly the same buffers: high bonus (encourages fusion)
        - For common buffers: score based on overlap ratio
          - overlap_ratio = common_buffer_size /
            max(node1_total_reads, node2_total_reads)
          - If overlap_ratio > threshold (e.g., 0.5): give proportional score
          - If overlap_ratio < threshold: minimal/no score (not worth fusing)

        Note on dynamic shapes:
        - When deps have unbacked symbols (dynamic shapes), dep_size_hint returns 0
        - In this case, we use count * 10 as a proxy for size
        - This ensures fusion still works for models with dynamic batch sizes

        Note on multiple deps from same buffer:
        - A node may have multiple MemoryDep entries for the same buffer name
          (e.g., 4 split reads from arg0_1 at different indices)
        - We sum ALL dep sizes for each buffer, not just take max
        - This ensures overlap ratio is calculated correctly when nodes read
          multiple slices from the same underlying buffer
        r	  c                ó8   •— ‰j                  | «      }|dkD  r|S ‰S rÅ   r8  )r¼   r¸   ÚFALLBACK_DEP_SIZErk   s     €€rl   Úget_dep_sizezFScheduler._score_fusion_memory_by_buffer_overlap.<locals>.get_dep_sizeK  s%   ø€ Ø×%Ñ% cÓ*ˆDØ !š8�4Ð:Ð):Ð:rn   c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   zCScheduler._score_fusion_memory_by_buffer_overlap.<locals>.<genexpr>O  ry  rµ  c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   zCScheduler._score_fusion_memory_by_buffer_overlap.<locals>.<genexpr>P  ry  rµ  r   c              3  ó.   •K  — | ]  } ‰|«      –— Œ y ­wrw   r~   ©r’   r¼   rP  s     €rl   r”   zCScheduler._score_fusion_memory_by_buffer_overlap.<locals>.<genexpr>Y  ó   øè ø€ ò $
Ø"%‰L˜×ñ$
ùr>  c              3  ó.   •K  — | ]  } ‰|«      –— Œ y ­wrw   r~   rT  s     €rl   r”   zCScheduler._score_fusion_memory_by_buffer_overlap.<locals>.<genexpr>\  rU  r>  c              3  óJ   •K  — | ]  }|j                   ‰v r
 ‰|«      –— Œ y ­wrw   r³  ©r’   r¼   Úcommon_namesrP  s     €€rl   r”   zCScheduler._score_fusion_memory_by_buffer_overlap.<locals>.<genexpr>g  ó*   øè ø€ ò %
àØ�x‰x˜<Ñ'ñ ˜×ñ%
ùó   ƒ #c              3  óJ   •K  — | ]  }|j                   ‰v r
 ‰|«      –— Œ y ­wrw   r³  rX  s     €€rl   r”   zCScheduler._score_fusion_memory_by_buffer_overlap.<locals>.<genexpr>l  rZ  r[  )r¼   r3   r†   r  )r   r°   r±   rÙ   r¶  r(   Úmin_overlap_ratio)rk   r�   r‚   Únode1_read_namesr‚  Únode1_total_read_sizeÚnode2_total_read_sizeÚmax_total_read_sizeÚnode1_common_read_sizeÚnode2_common_read_sizeÚcommon_read_buffer_sizeÚoverlap_ratiorO  rY  rP  s   `           @@@rl   r;  z0Scheduler._score_fusion_memory_by_buffer_overlap+  sQ  û€ ð< Ðö	;ô &Ñ%R¸%×:KÑ:K×:QÑ:QÔ%RÓRÐÜ%Ñ%R¸%×:KÑ:K×:QÑ:QÔ%RÓRÐð (Ð*:Ñ:ˆáØô !$ó $
Ø).×):Ñ):×)@Ñ)@ô$
ó !
Ðô !$ó $
Ø).×):Ñ):×)@Ñ)@ô$
ó !
Ðô "Ð"7Ð9NÓOÐØ !Ò#Øô
 "%ô %
à×(Ñ(×.Ñ.ô%
ó "
Ðô
 "%ô %
à×(Ñ(×.Ñ.ô%
ó "
Ðô #&Ð&<Ð>TÓ"UÐð 0Ð2EÑEˆð
 (5¼×8PÑ8PÒ'PÐ#ð	
ØVWð	
rn   c                óÀ  — t        |«      dk(  r|S i }|D ]…  \  }}|j                  «       |j                  «       k(  sJ ‚|j                  «       }t        | j                  |«      j	                  ||«      «      }||vr	||fg||<   Œp||   j                  ||f«       Œ‡ t        |j                  «       t        j                  d«      ¬«      d   }t        |«      dkD  sJ ‚|S )Nr   r$  r   )
r©   rÞ   r  rc  Úget_fusion_pair_priorityrÀ   r‰  r�  r  r  )rk   r6  Ú"possible_fusions_group_by_priorityr�   r‚   r  Úfusion_pair_priorityÚ&possible_fusions_with_highest_prioritys           rl   rQ  z4Scheduler.get_possible_fusions_with_highest_priority  s  € ô
 ÐÓ  AÒ%Ø#Ð#ð ð 	+ð -ò 	‰LˆE�5Ø×#Ñ#Ó%¨×)9Ñ)9Ó);Ò;Ð;Ð;Ø×%Ñ%Ó'ˆFÜ#&Ø× Ñ  Ó(×AÑAÀ%ÈÓOó$Ð ð $Ð+MÑMà˜E�NðLÐ2Ð3GÒHð 3Ð3GÑH×OÑOØ˜E�Nõð	ô 25Ø.×4Ñ4Ó6¼H×<OÑ<OÐPQÓ<Rô2
à
ñ2Ð.ô Ð9Ó:¸QÒ>Ð>Ð>Ø5Ð5rn   c                óB   — t        j                  j                  | g|¢­Ž S )z-
        Shim for list.sort(key=...)
        )rZ   r  Úscore_fusionrO  s     rl   rR  zScheduler.score_fusion_keyŸ  s   € ô �y‰y×%Ñ% dÐ3¨UÒ3Ð3rn   c                óô   — t        t        j                  j                  «       «      }t	        | j
                  «      D ]9  }|j                  || j                  «       |j                  |j                  «       Œ; y)zg
        Populate node.last_usage recursively (also for the nodes within a FusedSchedulerNode)
        N)
r   rZ   rœ   r•  r«   rÒ  rÏ  rÊ  r  r`  )rk   rÍ  r�   s      rl   r¡  zScheduler.compute_last_usage§  s]   € ô
 )¬¯©×)AÑ)AÓ)CÓDÐä˜TŸZ™ZÓ(ò 	8ˆDØ×ÑÐ 3°T×5LÑ5LÔMØ×&Ñ& t§¡Õ7ñ	8rn   c                óÞ  — t        | j                  t        j                  j                  z
  t        j                  j
                  j                  z
  «      D �]z  }|| j                  v rT| j                  |   }|j                  «       sŒ2t        j                  j
                  j                  |j                  «       Œf|t        j                  j                  v sŒƒt        j                  j                  |   }t        |t        j                  «      r*t        j                  j
                  j                  |«       Œät        |t        j                  t        j                   f«      r�Œ|j"                  }t        |t        j$                  «      r|j'                  «       sJ ‚t        j                  j
                  j                  |j"                  «       �Œ} | j                  j)                  «        y)z*Free any buffers that are no longer neededN)r)  r§  rZ   rœ   r?  r:  Úfreedr@  rI  Úcodegen_freer�   r–  rŒ   r+   r‡  rí  rî  rÔ   r™  Úis_input_bufferr@  )rk   r²   rº   r‡  Ústorages        rl   Úfree_bufferszScheduler.free_buffers²  sV  € äØ×%Ñ%Ü�g‰g×%Ñ%ñ&ä�g‰g×"Ñ"×(Ñ(ñ)ó
ó 	DˆDð
 �t×'Ñ'Ñ'Ø×&Ñ& tÑ,�Ø—<‘<•>Ü—G‘G×(Ñ(×5Ñ5°c·h±hÕ?ØœŸ™×-Ñ-Ò-Ü—g‘g×*Ñ*¨4Ñ0�Ü˜c¤2×#5Ñ#5Ô6Ü—G‘G×(Ñ(×5Ñ5°cÕ:Ü ¤b×&7Ñ&7¼×9MÑ9MÐ%NÔOÙà!Ÿh™h�Gä" 7¬B¯M©MÔ:¸w×?VÑ?VÔ?XðØXä—G‘G×(Ñ(×5Ñ5°g·l±lÖCð)	Dð, 	×!Ñ!×'Ñ'Õ)rn   c                ó‚   — | j                   j                  «       D ]  }|j                  «        Œ | j                  «        y rw   )rl  r¹   Úflushrs  )rk   rQ  s     rl   ru  zScheduler.flushÌ  s3   € Ø—}‘}×+Ñ+Ó-ò 	ˆGØ�M‰M�Oð	à×ÑÕrn   c                óv  — t        |t        t        f«      sJ ‚t        d   dxx   dz  cc<   t	        j
                  t        d¬«      «      5  |j                  «        |j                  «        d d d «       |j                  t        j                  j                  «       | j                  «        y # 1 sw Y   ŒCxY w)Nr›  Úextern_callsr   F)Úincrease_kernel_count)rŒ   r�  rV  r   rZ   Úset_kernel_handlerr0   rL  rÑ  r;  rœ   r:  rs  )rk   Úscheduler_nodes     rl   Úcodegen_extern_callzScheduler.codegen_extern_callÑ  sŸ   € ô ØÜ&Ô(LÐMô
ð 	
ð 
ô 	�Ñ˜^Ó,°Ñ1Ó,Ü×!Ñ!¤&¸uÔ"EÓFñ 	&Ø×0Ñ0Ô2Ø×#Ñ#Ô%÷	&ð 	×ÑœqŸw™w×3Ñ3Ô4Ø×ÑÕ÷		&ð 	&ús   Á!B/Â/B8c                óP  — t        |j                  «      r|j                  €
J |› d�«       ‚t        j                  j                  |«       t        |j                  «      }|€t        d|j                  › �«      ‚t        «       sœ|j                  dk(  rLt        j                  j                  |«      x}j                  dk  rt        |t        j                  «       «      ‚t        |j                  «      r,|j                  dk(  st!        t        j                  «       «      ‚ || «      S )Nz( should have been normalized in loweringzUnsupported device type: rÐ   é   rŸ  )rU   r¶   r³   rZ   rœ   Úadd_device_infor/   rU  r&   rŸ  rÐ   Úget_device_propertiesÚmajorr7   ÚinspectÚcurrentframer8   )rk   r  Údevice_schedulingr  s       rl   Úcreate_backendzScheduler.create_backendä  sï   € Ü˜&Ÿ+™+Ô&¨&¯,©,Ð*Bð 	
ØˆhÐ>Ð?ó	
ÐBô 	
�‰×Ñ Ô'ä5°f·k±kÓBÐØÐ$ÜÐ!:¸6¿;¹;¸-ÐHÓIÐIäŒ|à—‘˜vÒ%Ü%*§Z¡Z×%EÑ%EÀfÓ%MÐM�\×TÑTÐWXÒXä(¨´w×7KÑ7KÓ7MÓNÐNÜ˜Ÿ™Ô$¨V¯[©[¸EÒ-AÜ#¤G×$8Ñ$8Ó$:Ó;Ð;á  Ó&Ð&rn   c                ó€   — |€J ‚|| j                   vr| j                  |«      | j                   |<   | j                   |   S rw   )rl  r„  r  s     rl   rc  zScheduler.get_backendù  sB   € ØÐ!Ð!Ð!Ø˜Ÿ™Ñ&Ø$(×$7Ñ$7¸Ó$?ˆD�M‰M˜&Ñ!Ø�}‰}˜VÑ$Ð$rn   c                ó”  ‡ — dˆ fd„}|j                  «       D ��ci c]8  }|j                  �*|j                  j                  «       D ]  } ||«      |fd “Œ Œ: }}}t        |j	                  «       «      }|rMt        |t        j                  d«      ¬«      \  }}t        j                  j                  j                  |«       y y c c}}w )Nc                óâ   •— | ‰j                   vrL‰j                   j                  t        | j                  j                  «      D �� ci c]  \  }} | |“Œ
 c} }«       ‰j                       S c c} }w rw   )r¨  r  rÑ  rœ   rÒ  )r  rÏ  rk   s     €rl   Ú	get_orderz*Scheduler.enter_context.<locals>.get_order   s\   ø€ Ø˜×,Ñ,Ñ,Ø×$Ñ$×+Ñ+¼iÈÏÉÏÉÓ>V×,W±d°a¸¨Q°©TÓ,WÔXØ×'Ñ'¨Ñ*Ð*ùó -Xs   ÁA+
r   r$  )r  ztorch.fx.Noder†   r  )r–   r�   rW  rÿ   r   r¶  r  r  rZ   rœ   r:  Úenter_context)rk   r�   rˆ  r  r  r_  r  Úlasts   `       rl   r‰  zScheduler.enter_contextÿ  sº   ø€ õ	+ð —^‘^Ó%÷
àØ�v‰vÐ!Ø—V‘V×'Ñ'Ó)ò	
ð ñ �q‹\˜1Ð˜tÑ#ð
Øð
ˆñ 
ô �w—|‘|“~Ó&ˆÙÜ˜'¤x×':Ñ':¸1Ó'=Ô>‰GˆAˆtÜ�G‰G× Ñ ×.Ñ.¨tÕ4ð ùó
s   ›=Cc                ó¾   ‡— 	 | j                   |   j                  }t        ˆfd„|D «       «      xr || j                  vxr || j
                  vS # t        $ r Y yw xY w)NFc              3  ó^   •K  — | ]$  }|j                   xs |j                  «       ‰v –— Œ& y ­wrw   )r  r  )r’   r.  r€  s     €rl   r”   zAScheduler.can_buffer_be_removed_through_fusion.<locals>.<genexpr>  s)   øè ø€ ÒVÈ�—‘ÒC §¡£Ð3CÐ CÓCÑVùs   ƒ*-)r@  r  ÚKeyErrorr•   rg  rÊ  )rk   r²   r€  r  s     ` rl   r~  z.Scheduler.can_buffer_be_removed_through_fusion  sn   ø€ ð	Ø×$Ñ$ TÑ*×0Ñ0ˆEô ÓVÐPUÔVÓVò 4Ø˜D×1Ñ1Ð1ò4à˜D×3Ñ3Ð3ð	
øô ò 	Ùð	ús   ƒA Á	AÁAc                ó  — |j                   }t        |t        j                  j                  j
                  «      rk|j                  x}r]t        |«      \  }}|t        j                  v s|t        j                  v r+t        |t        j                  j                  «      sJ ‚d|› �S t        j                  j                  j                  j                  st        j                  €yt        |t         «      r)|j"                  D ]  }| j%                  |«      }|sŒ|c S  y|j                   €J ‚|j'                  «       s|j)                  «       › d�S t        |j                   t        j*                  «      ryt        |j                   t        j,                  «      ryt/        |j                   dd«      ryt1        |j                   «      ry	| j3                  |«      x}r|S t        j                  j4                  rt7        |«      ry
y)zŠ
        Return the reason why we should partition the inductor graph on this node,
        or None if the node is cudagraphable.
        zcustom partition op: Nz6partition includes all ops when cudagraphs is disabledz opszDeviceCopy opszConditional opsÚunbacked_bindingszunbacked binding opszCUDAGraph-unsafe custom opszdynamic shape ops)r�   rŒ   rŸ  r   r+   rD  r�  rP   r(   Úcustom_should_partition_opsÚ_opsÚ
OpOverloadrÒ   r^   rH   rN  rµ   r·   Úshould_partitionrU   rÞ   Ú
DeviceCopyrL  r�  rT   Ú&_uses_cudagraph_unsafe_unbacked_symintÚcudagraph_skip_dynamic_graphsr2  )rk   r�   r5  r  Úop_overload_packet_nameÚop_overload_namerø  r  s           rl   r“  zScheduler.should_partition  s¦  € ð —)‘)ˆÜ�gœuŸ™×1Ñ1×@Ñ@ÔAØ×%Ñ%Ð%ˆBÐ%ä8DÀRÓ8HÑ5Ð#Ð%5à'¬6×+MÑ+MÑMØ#¤v×'IÑ'IÑIä! "¤e§j¡j×&;Ñ&;Ô<Ð<Ð<Ø.Ð/?Ð.@ÐAÐAô —‘×&Ñ&×-Ñ-×8Ò8Ü6×>Ñ>ÐFàKä�dÔ.Ô/ØŸ™ò "�Ø×.Ñ.¨uÓ5�ÚØ!’Mð"ð à�y‰yÐ$Ð$Ð$à�{‰{Œ}Ø—o‘oÓ'Ð(¨Ð-Ð-ä�d—i‘i¤§¡Ô/Ø#ä�d—i‘i¤§¡Ô0Ø$ä�4—9‘9Ð1°4Ô8Ø)ä! $§)¡)Ô,Ø0à×@Ñ@ÀÓFÐFˆ6ÐFØˆMô �=‰=×6Ò6Ü-¨dÔ3Ø*àrn   c                óT  — t        «       }t        j                  s|S | j                  D ]ü  }|j                  }|€Œt        |t        j                  j                  j                  «      sŒA|j                  }|€ŒPt        |«      \  }}|t        j                  vr|t        j                  vrŒƒ|j                  «       D ]g  }t        j                  j                  j!                  |«      }t#        |t$        j&                  t$        j(                  f«      sŒW|j+                  |«       Œi Œþ |S )zc
        Collect output unbacked symints from ops in config.cudagraph_unsafe_unbacked_ops.
        )r   r(   Úcudagraph_unsafe_unbacked_opsrÒ  r�   rŒ   rŸ  r   r+   rD  r�  rP   rü  rZ   rœ   r�   rž   r$   r%   ÚUNBACKED_INTÚUNBACKED_FLOATrÜ  )rk   Úunsafe_symintsr�   r5  r  r—  r˜  Úsyms           rl   Ú&_get_cudagraph_unsafe_unbacked_symintsz0Scheduler._get_cudagraph_unsafe_unbacked_symints]  s  € ô
 4>³<ˆä×3Ò3Ø!Ð!à—J‘Jò 	,ˆDØ—i‘iˆGØˆØä˜g¤u§¡×'9Ñ'9×'HÑ'HÔIØà×$Ñ$ˆBØˆzØä8DÀRÓ8HÑ5Ð#Ð%5à'¬v×/SÑ/SÑSØ$¬F×,PÑ,PÑPàà×7Ñ7Ó9ò ,�Ü—g‘g×&Ñ&×/Ñ/°Ó4�Ü! #¬×(9Ñ(9¼4×;NÑ;NÐ'OÕPØ"×&Ñ& sÕ+ñ,ð'	,ð0 Ðrn   c                óÜ   — | j                  «       }|sy t        |«      }|D ]I  }t        j                  j                  j                  |«      }|j                  D ]  }||v sŒd|› �c c S  ŒK y )Nz'uses cudagraph-unsafe unbacked symint: )rŸ  r2  rZ   rœ   r�   rž   r"   )rk   r�   r�  Únode_symbolsrž  Úsimplified_symÚfree_syms          rl   r•  z0Scheduler._uses_cudagraph_unsafe_unbacked_symint�  s~   € ð ×DÑDÓFˆÙØä5°dÓ;ˆàò 	PˆCÜŸW™W×-Ñ-×6Ñ6°sÓ;ˆNØ*×7Ñ7ò P�Ø˜~Ò-ØDÀXÀJÐOÔOñPð	Pð rn   c                óà   — i }|j                  t        j                  j                  «       | j                  D ]3  }|j
                  j                  «       D ]  \  }}|j                  ||<   Œ Œ5 |S )z~
        Return a mapping from name strings to the corresponding graph inputs or
        base scheduler node outputs.
        )r  rZ   rœ   r–  rÒ  rl  r�  r�   )rk   ru  r�   r²   Úscheduler_buffers        rl   Úget_name_to_nodeszScheduler.get_name_to_nodes’  sr   € ð PRˆØ×ÑœAŸG™G×0Ñ0Ô1à—J‘Jò 	;ˆDØ*.×*>Ñ*>×*DÑ*DÓ*Fò ;Ñ&�Ð&Ø%5×%:Ñ%:�˜TÒ"ñ;ð	;ð Ðrn   c           	     óÌ  — t        t        j                  j                  «      D ��ci c]  \  }}||“Œ
 }}}t        t        j                  j	                  «       «      D ��ci c]  \  }}||“Œ
 }}}g t        j                  _        t        |«      D ]Å  \  }}|j                  rŒg }|j                  D ]"  }|j                  |j                  |«      «       Œ$ g }	|j                  D ]0  }
|	j                  |j                  |
j                  «       «      «       Œ2 t        j                  j
                  j                  t        |||	|j                  «      «       ŒÇ yc c}}w c c}}w )z‚
        computes a mapping from partition input/output indices to graph input/output
        indices for each partition.
        N)rÑ  rZ   rœ   r–  r•  Úpartition_mapsÚskip_cudagraphÚinput_nodesrÀ   rÈ  Úoutput_nodesr  rQ   Úconstant_names)rk   Ú
signaturesræ  r²   Úname_to_graph_input_indexÚname_to_graph_output_indexÚpartition_idÚ	signatureÚinput_mappingÚoutput_mappingr�   s              rl   Úcompute_graph_partition_mapsz&Scheduler.compute_graph_partition_maps¢  sT  € ô (1´·±×1EÑ1EÓ'F÷%
Ù#˜#˜tˆD�#‰Ið%
Ð!ñ %
ô (1´·±×1IÑ1IÓ1KÓ'L÷&
Ù#˜#˜tˆD�#‰Ið&
Ð"ñ &
ð "$Œ�‰ÔÜ'0°Ó'<ò 	Ñ#ˆL˜)Ø×'Ò'ð
 àˆMØ!×-Ñ-ò J�Ø×$Ñ$Ð%>×%BÑ%BÀ4Ó%HÕIðJð  ˆNØ!×.Ñ.ò W�Ø×%Ñ%Ð&@×&DÑ&DÀTÇ]Á]Ã_Ó&UÕVðWô �G‰G×"Ñ"×)Ñ)Ü!Ø Ø!Ø"Ø×,Ñ,ó	õñ!	ùó%
ùó&
s   §EÁ!E c                ó¸  ‡— 	 	 	 	 dd„Š	 	 	 	 dd„} t        «       j                  d„ |D «       Ž } |j                  ˆfd„|j                  «       D «       Ž   ||«      }t        «       }|D ]F  }t        j
                  j                  j                  |«      }|j                  |j                  «       ŒH t        t        |t        j                  d«      ¬«      «      S )	ai  
        Returns all symbol inputs which are required to be in scope to successfully
        perform codegen for this graph partition, including:
        - free symbols used in partition nodes
        - free symbols in partition input/node shapes, strides, and offsets. This is needed
          for recording cudagraphs for tensors with dynamic shapes.
        c                óÂ   — t        | t        j                  «      r
t        «       S t        | t        j                  «      rt        | «      S t        dt        | «      › �«      ‚)zW
            Gets symbols used in input node shapes, strides, and offsets.
            zUnsupported input node type: )rŒ   r+   r‡  r   rª  r-  r+  r¶   r—   s    rl   Úget_input_node_symbolszKScheduler.get_graph_partition_symbol_inputs.<locals>.get_input_node_symbolsØ  sN   € ô ˜$¤× 2Ñ 2Ô3ä!“|Ð#Ü˜D¤"§)¡)Ô,Ü)¨$Ó/Ð/ô *Ð,IÌ$ÈtË*ÈÐ*VÓWÐWrn   c                ó&   — t        d„ | D «       «      S )zÌ
            Filters a set of symbols that are required for codegen. Skip symbols
            that are always internal to kernels, such as SymT.TMP, SymT.INDEX,
            and SymT.R0_INDEX.
            c              3  ó°   K  — | ]N  }t        |t        j                  t        j                  t        j                  t        j
                  f«      r|–— ŒP y ­wrw   )r$   r%   ÚSIZEÚFLOATr›  rœ  ©r’   rt  s     rl   r”   zVScheduler.get_graph_partition_symbol_inputs.<locals>.filter_symbols.<locals>.<genexpr>ð  sH   è ø€ ò àÜ!ØäŸ	™	ÜŸ
™
Ü×)Ñ)Ü×+Ñ+ð	ôô ñùs   ‚AAr   )Úsymbolss    rl   Úfilter_symbolszCScheduler.get_graph_partition_symbol_inputs.<locals>.filter_symbolsè  s   € ô ñ à ôó ð rn   c              3  ó2   K  — | ]  }t        |«      –— Œ y ­wrw   r1  rü  s     rl   r”   z>Scheduler.get_graph_partition_symbol_inputs.<locals>.<genexpr>ÿ  s   è ø€ ÒI°tÔ,¨T×2ÑIùr3  c              3  ó4   •K  — | ]  \  }} ‰|«      –— Œ y ­wrw   r~   )r’   r  r�   r·  s      €rl   r”   z>Scheduler.get_graph_partition_symbol_inputs.<locals>.<genexpr>  s   øè ø€ ÒN©w¨q°$Ñ$ T×*ÑNùs   ƒr²   r$  )r�   z+ir.IRNode | sympy.Expr | ir.TorchBindObjectr†   úOrderedSet[sympy.Symbol])r½  rÁ  r†   rÁ  )r   rá  r  r�  rZ   rœ   r�   rž   r"   r)  r  Ú
attrgetter)	rk   Ú	partitionrª  r¾  Úcandidate_symbolsrß  rt  Úsymplified_sr·  s	           @rl   Ú!get_graph_partition_symbol_inputsz+Scheduler.get_graph_partition_symbol_inputsË  sã   ø€ ð	XØ=ð	Xà%ó	Xð 	Ø-ð	à%ó	ð, 7I´j³l×6HÑ6HÙI¸yÔIð7
Ðð 	!Ð× Ñ ÛN¸+×:KÑ:KÓ:MÔNñ	
ñ +Ð+<Ó=Ðä(2«ˆØ"ò 	2ˆAÜŸ7™7×+Ñ+×4Ñ4°QÓ7ˆLà�J‰J�|×0Ñ0Õ1ð	2ô
 œ& ¬(×*=Ñ*=¸fÓ*EÔFÓGÐGrn   c           
     óŽ  ‡ ‡— g }t        t        j                  j                  «       «      }‰ j	                  «       }dˆˆ fd„Št        t        |«      t        |«      «      D �]2  \  }}t        «       }|D ]+  }	|j                  |	j                  j                  «       «       Œ- |j                  |«      }
t        j                  j                  |D �	cg c]  }	|	j                  ‘Œ c}	«      }t        |j                  |j                   z  D �cg c]  }t#        |t$        «      s|j&                  ‘Œ  c}«      |z
  }t        ˆ fd„|D «       «      }t        «       }|D ]  }	|j                  |	j(                  «       Œ ||z
  D �cg c]  }||v r|‘Œ
 }}|j                  |«       |D �ci c]  }||v r|||   “Œ }}|D �ci c]  }||v r|||v “Œ }}|D �cg c]  }||v r||vr|‘Œ }}|
j                  |«       t        ˆ fd„|
D «       «      }
|
D �cg c]  } ‰|«      s||   ‘Œ }}|D �cg c]!  }|t        j                  j*                  v sŒ |‘Œ# }}‰ j-                  ||«      }t/        ||||||«      }|j1                  |«       |j3                  ||
z
  «      }�Œ5 |ddd…   S c c}	w c c}w c c}w c c}w c c}w c c}w c c}w c c}w )z¡
        Gets signature for each graph partition, including input nodes, output nodes, and
        whether deallocating an input within graph partition.
        c                óÚ   •— ‰j                   j                  | d«      }|€yt        |j                  j                  t
        «      r'‰j                  j                  | d«      x}r ‰|«      S yy)zì
            Checks if buf_name resolves to a NoneLayout buffer (following mutation_real_name).
            Buffers with NoneLayout are not allocated so graph partition should not
            take them as inputs or outputs.
            NFT)r@  rÈ  rŒ   r�   r'  r?   rÊ  )r  rº   r«  Úis_unallocated_bufferrk   s      €€rl   rÉ  zFScheduler.get_graph_partition_signature.<locals>.is_unallocated_buffer  sh   ø€ ð ×"Ñ"×&Ñ& x°Ó6ˆCàˆ{Øä˜#Ÿ(™(Ÿ/™/¬:Ô6ð !%× 7Ñ 7× ;Ñ ;¸HÀdÓ KÐK�9ÐKÙ0°Ó;Ð;ààrn   c              3  óV   •K  — | ]   }‰j                   j                  ||«      –— Œ" y ­wrw   ©rÊ  rÈ  ©r’   r²   rk   s     €rl   r”   z:Scheduler.get_graph_partition_signature.<locals>.<genexpr>N  ó,   øè ø€ ò /àð ×'Ñ'×+Ñ+¨D°$×7ñ/ùrô   c              3  óV   •K  — | ]   }‰j                   j                  ||«      –— Œ" y ­wrw   rË  rÌ  s     €rl   r”   z:Scheduler.get_graph_partition_signature.<locals>.<genexpr>x  rÍ  rô   Nr÷   )r  r  r†   rv   )r   rZ   rœ   r•  r¦  r±  r«   r  rl  r   rl  r*   rß  rà  r°   r±   r�  rŒ   r6   r²   r`  rp  rÆ  r<   rÀ   rá  )rk   Ú
partitionsÚskip_cudagraphsr­  Úunmet_output_namesru  rÃ  r©  Úoutput_namesr�   Úreturned_output_namesr°   rJ  Úpartition_input_namesr§  r²   Úextra_input_namesrª  Úinput_deallocationÚextra_output_namesr«  r¬  Úsymbol_inputsÚpartition_signaturerÉ  s   `                       @rl   Úget_graph_partition_signaturez'Scheduler.get_graph_partition_signature  s^  ù€ ð ˆ
ä'¬¯©×(@Ñ(@Ó(BÓCÐØ×-Ñ-Ó/ˆö	ô, *-Ü�ZÓ ¤(¨?Ó";ó*
ó g	Ñ%ˆI�~ô -7«LˆLà!ò A�Ø×#Ñ# D×$8Ñ$8×$=Ñ$=Ó$?Õ@ðAð %1×$=Ñ$=Ð>PÓ$QÐ!ô '×1Ñ1×<Ñ<Ø.7Ö8 d�×!Ó!Ò8óˆKô ð "-×!2Ñ!2°[×5GÑ5GÑ!GöàÜ)¨!¬WÔ5ð Ÿ›òóð ñð "ô %/ó /à1ô/ó %Ð!ô
 5?³LÐ Ø!ò =�Ø$×+Ñ+¨D¯O©OÕ<ð=ð 2°LÑ@ö!àØ˜<Ñ'ò ð!Ðð !ð
 "×(Ñ(Ð):Ô;ð 2öàØ˜<Ñ'ð �l 4Ñ(Ñ(ðˆKð ð 2ö"àØ˜<Ñ'ð �dÐ2Ð2Ñ2ð"Ðð "ð 2ö"àØ˜<Ñ'¨DÐ8LÑ,Lò ð"Ðð "ð "×(Ñ(Ð);Ô<ä$.ó /à1ô/ó %Ð!ð 2öàÙ,¨TÔ2ð ˜TÓ"ðˆLð ð "7öØ¸$Ä!Ç'Á'×BSÑBSÒ:S’ðˆNð ð !×BÑBØ˜;óˆMô #:ØØØØ"ØØó#Ðð ×ÑÐ1Ô2à!6×!<Ñ!<Ø"Ð%:Ñ:ó"ÒðKg	ðR ™$˜B˜$ÑÐùòy 9ùòùò*!ùòùò
"ùò"ùòùòs6   ÃJ
Ä
#J$
Å>J)Æ"J.Æ9J3ÇJ8ÈJ=È%!KÉKc                óŠ  — |j                   j                  «       D ��ci c]$  \  }}|t        j                  j                  vr||“Œ& }}}|j
                  j                  «       D ��ci c]$  \  }}|t        j                  j                  vr||“Œ& }}}|j                  D �cg c].  }|j                  «       t        j                  j                  vr|‘Œ0 }}|j                  D �cg c]   }|t        j                  j                  vr|‘Œ" }	}t        |j                  ||||j                  |	«      S c c}}w c c}}w c c}w c c}w )z¡
        Updates the partition signature by removing buffers specified in
        V.graph.removed_buffers. See [Note: Removed Graph Partition Arguments]
        )rª  r�  rZ   rœ   r?  rÖ  r«  Úmaybe_get_namer¬  r<   rØ  r©  )
rk   r±  r²   r]  rª  r  rÖ  r�   r«  r¬  s
             rl   Ú.clean_removed_buffer_from_partition_signaturesz8Scheduler.clean_removed_buffer_from_partition_signaturesœ  sK  € ð !*× 5Ñ 5× ;Ñ ;Ó =÷
á��fØœ1Ÿ7™7×2Ñ2Ñ2ð �&‰Lð
ˆñ 
ð '×9Ñ9×?Ñ?ÓA÷
á��cØœ1Ÿ7™7×2Ñ2Ñ2ð �#‰Ið
Ðñ 
ð "×.Ñ.ö
àØ×"Ñ"Ó$¬A¯G©G×,CÑ,CÑCò ð
ˆð 
ð "×0Ñ0ö
àØœ1Ÿ7™7×2Ñ2Ñ2ò ð
ˆð 
ô
 'Ø×#Ñ#ØØØØ×$Ñ$Øó
ð 	
ùó)
ùó

ùò

ùò

s   ž)D/Á')D5Â!3D;Ã$%E c                óp  ‡ ‡‡	‡
‡‡‡— ddl Š	t        «       Šg Šg Št        |«      D ��ci c]  \  }}||“Œ
 c}}Šdˆˆ	ˆˆˆ fd„Š
dˆ
ˆfd„}|D ]5  }t        |j                  j
                  «      ‰|<   ‰|   dk(  sŒ. ‰
|«       Œ7 g }d}|t        |«      k  r�‰s‰r}‰r0‰	j                  ‰«      \  }}|j                  |«        ||«       ‰rŒ0‰r0‰	j                  ‰«      \  }}|j                  |«        ||«       ‰rŒ0|dz  }|t        |«      k  r‰rŒz‰rŒ}|t        |«      kD  rt        d«      ‚|S c c}}w )aœ  
        Reorder nodes to minimize the number of partitions via a bfs
        topological sort. This is the optimal reordering such that the
        number of partitions cannot be reduced further. This may be
        sub-optimal for other metrics such as peak memory. This does not
        change relative orders of two cudagraphable nodes, nor the
        relative order of two non_cudagraphable nodes.
        r   Nc                ó€   •— ‰|    | f}‰j                  | «      r‰j                  ‰|«       y ‰j                  ‰|«       y rw   )r“  Úheappush)r�   Únode_with_indexÚcudagraphable_nodesÚheapqÚnode_to_indexÚnon_cudagraphable_nodesrk   s     €€€€€rl   Úinsert_pending_nodeszHScheduler.reorder_for_minimizing_partition.<locals>.insert_pending_nodesÓ  s>   ø€ Ø,¨TÑ2°DÐ9ˆOØ×$Ñ$ TÔ*Ø—‘Ð6¸ÕHà—‘Ð2°OÕDrn   c                óŒ   •— | j                   j                  D ]*  }‰|   dkD  sJ ‚‰|xx   dz  cc<   ‰|   dk(  sŒ# ‰|«       Œ, y r=  )re  Ú
succ_nodes)r�   Ú	succ_noderæ  Únode_to_indegrees     €€rl   Úupdate_indegreezCScheduler.reorder_for_minimizing_partition.<locals>.update_indegreeÚ  sT   ø€ Ø!Ÿ]™]×5Ñ5ò 4�	Ø'¨	Ñ2°QÒ6Ð6Ð6Ø  Ó+¨qÑ0Ó+Ø# IÑ.°!Ó3Ù(¨Õ3ñ	4rn   r   z’
                Failed to schedule, while loop ran too long when
                reordering for minimizing the num of partitions
                ©r�   r_   r†   rU  )	rã  r°  rÑ  r©   re  Ú
pred_nodesÚheappoprÀ   rU  )rk   rÒ  ræ  r�   rë  r…  Ú	num_itersr  râ  rã  ræ  rê  rä  rå  s   `       @@@@@@rl   Ú reorder_for_minimizing_partitionz*Scheduler.reorder_for_minimizing_partitionÀ  sU  þ€ ó 	ä9=»ÐØCEÐØGIÐÜ4=¸eÓ4D×E¡y s¨D˜˜s™ÓEˆ÷	Eñ 	Eö	4ð ò 	+ˆDÜ%(¨¯©×)AÑ)AÓ%BÐ˜TÑ"Ø Ñ%¨Ó*Ù$ TÕ*ð	+ð
 -/ˆØˆ	Øœ#˜e›*Ò$Ù#Ñ':á)ØŸ-™-Ð(?Ó@‘��4Ø—‘ Ô%Ù Ô%ò *ñ
 &ØŸ-™-Ð(;Ó<‘��4Ø—‘ Ô%Ù Ô%ò &ð
 ˜‰NˆIð œ#˜e›*Ò$Ú#Ò':ð ”s˜5“zÒ!Üðóð ð ˆùó] Fs   ¨D2c           	     óX  — ddl m}m} t        t        j
                  j                  «       «      } ||| j                  | j                  t        t        j
                  j                  j                  «       «      |«      \  }}| j                  |«      } ||||«      \  }}	||dz  k  r|S |S )zx
        Reorder nodes to minimize the number of partitions if this only slightly
        increase peak memory.
        r   )Úestimate_peak_memoryÚprepare_planning_inforÉ  )r”  rò  ró  r   rZ   rœ   r•  r@  ró  r–  r   rð  )
rk   rÒ  rò  ró  r   Údefault_peak_memoryr5  Úreordered_nodesÚreorder_peak_memoryr  s
             rl   rŸ  z0Scheduler.maybe_reorder_for_minimizing_partition  s¨   € ÷ 	Hä"¤1§7¡7×#;Ñ#;Ó#=Ó>ˆá:OØØ×ÑØ×#Ñ#Ü”q—w‘w×+Ñ+×0Ñ0Ó2Ó3Øó;
Ñ7ÐÐ7ð ×?Ñ?ÀÓFˆÙ!5ØÐ7¸ó"
ÑÐ˜Qð
 Ð!4°sÑ!:Ò:Ø"Ð"àˆrn   c                ó  — g }g }g }dd„}|D ]n  }| j                  |«      du}|r*t        |j                  «      dk(  r|j                  |«       ŒB|r ||«      r|j                  |«       Œ^|j                  |«       Œp ||z   |z   S )a  
        Reorder a node if it should be partitioned and has simple dependency:
        1. move a partitioned node to the front if it has no dependency
        2. move a partitioned node to the back if it is only used by OutputNode
        3. otherwise do not reorder
        c                óŠ   — | j                  «       D ]0  }|j                  D ]  }t        |j                  t        «      rŒ  y Œ2 yrF  )rz  r  rŒ   r�   rG  )r�   rº   rH  s      rl   Úonly_output_userzPScheduler.reorder_for_partition_with_simple_dependency.<locals>.only_output_user.  sC   € Ø×'Ñ'Ó)ò %�ØŸ9™9ò %�CÜ% c§h¡h´
Õ;Ú$ñ%ð%ð rn   Nr   r	  )r“  r©   rq  rÀ   )rk   rÒ  ÚfrontÚmiddleÚbackrù  r�   r“  s           rl   r   z6Scheduler.reorder_for_partition_with_simple_dependency   s”   € ð *,ˆØ*,ˆØ(*ˆó	ð ò 	$ˆDØ#×4Ñ4°TÓ:À$ÐFÐÙ¤C¨×(?Ñ(?Ó$@ÀAÒ$EØ—‘˜TÕ"Ù!Ñ&6°tÔ&<Ø—‘˜DÕ!à—‘˜dÕ#ð	$ð �v‰~ Ñ$Ð$rn   c                ó€  — g }d}g }g }| j                   D ]S  }| j                  |«      du}|r)||k7  r$|j                  |«       |j                  |«       g }|}|j                  |«       ŒU |r"|j                  |«       |j                  |«       t        j                  j
                  }|dkD  rXt        t        ||«      «      D ]@  \  }\  }	}
|
rŒt        d„ |	D «       «      }||k  sŒ$d||<   t        j                  d|||«       ŒB | j                  ||¬«      }| j                  |«       | j                  ||«       ||fS )z’
        Given a list of BaseSchedulerNodes, split into a list of
        graph partitions and compute partition input/output signatures.
        TNr   c              3  ó@   K  — | ]  }t        |t        «      sd –— Œ y­w©r   N)rŒ   rA  r  s     rl   r”   z,Scheduler.graph_partition.<locals>.<genexpr>`  s#   è ø€ ò 'àÜ)¨!Ô-CÔDô ñ'ùrÊ  zFPartition %d has %d kernels, below minimum size %d, skipping cudagraph)rÏ  rÐ  )rÒ  r“  rÀ   r(   rÒ   Úcudagraph_min_partition_sizerÑ  r±  rÙ   Úcudagraphs_logr  rÚ  r´  Ú_log_graph_partitions)rk   rÏ  r©  Úcur_partitionrÐ  r�   Únode_should_partitionÚmin_sizerÏ  rÃ  ÚskipÚkernel_countr­  s                rl   r�  zScheduler.graph_partition@  sm  € ð +-ˆ
ØˆØ')ˆØˆØ—J‘Jò 	'ˆDØ$(×$9Ñ$9¸$Ó$?ÀtÐ$KÐ!Ù Ð3HÒ!HØ×!Ñ! -Ô0Ø×&Ñ& ~Ô6Ø "�à2ˆNØ× Ñ  Õ&ð	'ñ Ø×Ñ˜mÔ,Ø×"Ñ" >Ô2ô —=‘=×=Ñ=ˆØ�aŠ<Ü(1´#°jÀ/Ó2RÓ(Sò Ñ$�Ñ$�I˜tÚä#&ñ 'à!*ô'ó $�Lð
 $ hÓ.Ø-1˜¨Ñ*Ü&×,Ñ,ØdØØ(Ø$õ	ðð" ×7Ñ7Ø!°?ð 8ó 
ˆ
ð 	×)Ñ)¨*Ô5à×"Ñ" :¨zÔ:à˜:Ð%Ð%rn   c                ó`  — t         j                  t        j                  «      sy t	        d„ t
        j                  j                  D «       «      }|sy t        d„ |D «       «      }t        |«      |z
  }t         j                  dt        |«      ||«       t        t        ||«      «      D ]„  \  }\  }}t         j                  d|t        |«      |j                  rdndt        |j                  «      t        |j                  «      «       |j                  sŒm|D ]  }	| j!                  |	«       Œ Œ† y )Nc              3  ó2   K  — | ]  }t        |«      –— Œ y ­wrw   )rU   )r’   r  s     rl   r”   z2Scheduler._log_graph_partitions.<locals>.<genexpr>�  s   è ø€ ÒO°œV FŸ^ÑOùr3  c              3  ó:   K  — | ]  }|j                   rŒd –— Œ y­wrÿ  )r©  r¼  s     rl   r”   z2Scheduler._log_graph_partitions.<locals>.<genexpr>…  s   è ø€ Ò!P¨¸q×?OÓ?O¤!Ñ!Pùs   ‚”zCCreated %d graph partitions: %d cudagraphable, %d non-cudagraphablez3  Partition %d: %d nodes, %s, inputs=%d, outputs=%dznon-cudagraphableÚcudagraphable)r	  rÓ  rÔ  rÆ  ræ   rZ   rœ   Údevice_typesrÙ   r©   r  rÑ  r±  r©  rª  r«  Ú_log_non_cudagraphable_node)
rk   rÏ  r­  Úhas_gpu_deviceÚcudagraphable_countÚnon_cudagraphable_countrÏ  rÃ  r±  r�   s
             rl   r	  zScheduler._log_graph_partitionsw  s  € ô
 ×*Ñ*¬7¯=©=Ô9Øô ÑO¼!¿'¹'×:NÑ:NÔOÓOˆÙØä!Ñ!P¨ZÔ!PÓPÐÜ"% j£/Ð4GÑ"GÐÜ×ÑØQÜ�
‹OØØ#ô		
ô *3´3°zÀ:Ó3NÓ)Oò 	;Ñ%ˆAÑ%�	˜9Ü× Ñ ØEØÜ�I“Ø'0×'?Ò'?Ñ#À_Ü�I×)Ñ)Ó*Ü�I×*Ñ*Ó+ôð ×'Ó'à%ò ;�DØ×4Ñ4°TÕ:ñ;ñ	;rn   c                ó¢  — | j                  |«      }|sy|j                  «       }|j                  �|j                  j                  «       nd}d|› �g}t	        |j                  «      j
                  }|j                  d|› �«       |�F|j                  › ddj                  d„ |j                  D «       «      › d�}|j                  d|› �«       t        j                  d	|dj                  |«      «       |�Z|j                  j                  d
d«      }|r;|j                  «       j                  d«      D ]  }	t        j                  d|	«       Œ yyy)z)Log details for a non-cudagraphable node.Nzreason=zir=rƒ  rœ  c              3  ó2   K  — | ]  }t        |«      –— Œ y ­wrw   )r  )r’   rò  s     rl   r”   z8Scheduler._log_non_cudagraphable_node.<locals>.<genexpr>§  s   è ø€ Ò2P¸a´3°q·6Ñ2Pùr3  r~  zfx=z
    %s: %srP  r†  z         %s)r“  r  r�   r�  r¶   ry   rÀ   rX  rœ  r4  r	  r  rY  rÈ  ÚstripÚsplit)
rk   r�   r  ry  r   ÚpartsÚir_typeÚfx_strrP  Úlines
             rl   r	  z%Scheduler._log_non_cudagraphable_node›  s1  € à×&Ñ& tÓ,ˆÙØà—M‘M“Oˆ	Ø15·±Ð1F�$—)‘)×+Ñ+Ô-ÈDˆØ˜6˜(Ð#Ð$ˆÜ�t—y‘y“/×*Ñ*ˆØ�‰�s˜7˜)�_Ô%ØÐØŸ™Ð' q¨¯©Ñ2PÀ7Ç<Á<Ô2PÓ)PÐ(QÐQRÐSˆFØ�L‰L˜3˜v˜h˜Ô(ä×Ñ˜\¨9°d·i±iÀÓ6FÔGð ÐØ!Ÿ,™,×*Ñ*¨=¸$Ó?ˆKÙØ'×-Ñ-Ó/×5Ñ5°dÓ;ò >�DÜ"×(Ñ(¨¸Õ=ñ>ð ð rn   c                óä   — t        d«      5  t        j                  j                  j                  r| j                  «       n| j                  | j                  «      	 cd d d «       S # 1 sw Y   y xY w)NzScheduler.codegen)r   rŸ  r   r(   r�  Ú_codegen_partitionsÚ_codegenrÒ  rj   s    rl   r;  zScheduler.codegen³  sX   € ÜÐ-Ó.ñ 	ô —?‘?×)Ñ)×9Ò9ð ×(Ñ(Ô*à—]‘] 4§:¡:Ó.ð÷	÷ 	ò 	ús   ŒAA&Á&A/c                ó^  — ddl m} t        j                  j                  }t        | j                  «      }t        j                  j                  «       5  t        j                  j                  dd|› �||¬«       | j                  |«       t        t        j                  j                  |«      sJ ‚| j                  |«      }|t        j                  j                  _        t        j                  j                  j                  «        t        j                  j                  }t        j                  j                  j                  t        j                  j                   «      \  }}ddd«       t        j                  j                  j#                  «       t        j                  j                  j%                  ||«       t        j                  j                  j&                  j)                  |j*                  D �	cg c]  }	|	j-                  «       ‘Œ c}	«       y# 1 sw Y   Œ·xY wc c}	w )z,Codegen a partition given its inputs/outputsr   )ÚSubgraphPythonWrapperCodegenTÚ
partition_)Úis_subgraphÚsubgraph_nameÚparent_wrapper_codeÚpartition_signaturesN)r7  r	  rZ   rœ   r:  rÐ  ro  Úset_current_wrapper_codeÚinit_wrapper_coder	  rŒ   rÝ  r"	  Úwrite_prefixr²   ÚgenerateÚis_inferenceÚdefine_subgraph_launcher_fnÚcodegen_partition_callÚ	allocatedr  r«  r  )
rk   rÃ  r±  r	  r!	  Úgraph_partition_idÚ
graph_nameÚpartition_coder  r�   s
             rl   Ú_codegen_partition_wrapperz$Scheduler._codegen_partition_wrapper»  s‹  € õ 	BäŸg™g×2Ñ2ÐÜ! $×"?Ñ"?Ó@Ðä�W‰W×-Ñ-Ó/ñ 	TÜ�G‰G×%Ñ%Ø Ø *Ð+=Ð*>Ð?Ø$7Ø%.ð	 &ô ð �M‰M˜)Ô$ô œaŸg™g×2Ñ2Ð4PÔQÐQÐQØ×KÑKÈIÓVˆIØ8AŒA�G‰G× Ñ Ô5Ü�G‰G× Ñ ×-Ñ-Ô/äŸ™Ÿ™ˆJÜ !§¡× 4Ñ 4× =Ñ =¼a¿g¹g×>RÑ>RÓ SÑˆN˜A÷/	Tô2 	
�‰×Ñ×8Ñ8¸À^ÔTä	�‰×Ñ×3Ñ3Ð4FÈ	ÔRÜ	�‰×Ñ×&Ñ&×-Ñ-Ø)2×)?Ñ)?Ö@ ˆT�]‰]�_Ò@õ	
÷9	Tð 	Tüò: As   ÁDHÇ?H*ÈH'c                óL   ‡ ‡‡— t         j                  dˆˆ ˆfd„«       } |«       S )Nc               3  ó¸  •K  — ‰j                  ‰ ‰«       ‰j                  ryt        ‰j                  j                  «      rZ‰j                  j                  €J d«       ‚t
        j                  j                  j                  ‰j                  j                  «       	 d –— ‰j                  rGt        ‰j                  j                  «      r(t
        j                  j                  j                  «        d ‰_        y # ‰j                  rGt        ‰j                  j                  «      r(t
        j                  j                  j                  «        d ‰_        w xY w­w)Núdevice should have an index)
Ú%update_graph_partition_default_devicerv  rK   r¶   r³   rZ   rœ   r:  Úcodegen_device_guard_enterÚcodegen_device_guard_exit)rÏ  rk   r­  s   €€€rl   Úctxz1Scheduler.use_default_device_context.<locals>.ctxé  s  øè ø€ à×6Ñ6°zÀ:ÔNØ×*Ò*Ô/@Ø×+Ñ+×0Ñ0ô0ð ×2Ñ2×8Ñ8ÐDð Ø1óÐDô —‘×$Ñ$×?Ñ?Ø×/Ñ/×5Ñ5ôð3Ûà×.Ò.Ô3DØ×/Ñ/×4Ñ4ô4ô —G‘G×(Ñ(×BÑBÔDØ.2�Õ+øð	 ×.Ò.Ô3DØ×/Ñ/×4Ñ4ô4ô —G‘G×(Ñ(×BÑBÔDØ.2�Õ+üs    ƒBEÂC; Â AEÃ;AEÅE)r†   zIterator[None])Ú
contextlibÚcontextmanager)rk   rÏ  r­  r5	  s   ``` rl   Úuse_default_device_contextz$Scheduler.use_default_device_contextæ  s&   ú€ ô 
×	"Ñ	"ö	3ó 
#ð	3ñ* ‹uˆrn   c                ó   — t        |«      dk(  r|d   j                  sy dd„}	 	 	 	 	 	 dd„}d }t        ||«      D ]  \  }}|j                  rŒ ||«      } n |€y t        ||«      D ]  \  }}|j                  sŒ |||«      rŒ y  || _        y )Nr   r   c                ó4   — | d   j                  «       }|€J ‚|S rÅ   r¶  )rÃ  Úpartition_devices     rl   Úget_cudagraph_partition_devicezWScheduler.update_graph_partition_default_device.<locals>.get_cudagraph_partition_device  s'   € Ø(¨™|×6Ñ6Ó8ÐØ#Ð/Ð/Ð/Ø#Ð#rn   c                ó@   — | D ]  }|j                  «       }||k7  sŒ y yrF  r¶  )rÃ  Útarget_devicer�   r  s       rl   Úall_on_target_devicezMScheduler.update_graph_partition_default_device.<locals>.all_on_target_device  s/   € ð "ò !�ØŸ™Ó*�Ø˜]Ó*Ù ð!ð rn   )rÃ  r`   r†   r6  )rÃ  r`   r>	  r6  r†   rv   )r©   r©  r±  rv  )rk   rÏ  r­  r<	  r?	  Úcudagraph_partition_devicerÃ  r±  s           rl   r2	  z/Scheduler.update_graph_partition_default_device  sÊ   € ô ˆz‹?˜aÒ¨
°1©×(DÒ(Dð ó	$ð
	Ø$ð	Ø5Að	àó	ð &*Ð"Ü$'¨
°JÓ$?ò 	Ñ ˆI�yØ×+Ó+Ù-KÈIÓ-VÐ*Ùð	ð &Ð-Øä$'¨
°JÓ$?ò 	Ñ ˆI�yØ×'Ó'Ñ0DØÐ5õ1ñ ð		ð 'AˆÕ#rn   c                ó  — | j                  «       \  }}t        |«      dkD  rt        d   dxx   t        |«      z  cc<   | j                  ||«      5  t	        ||«      D ]V  \  }}t        |«      dk\  sJ dt        |«      › �«       ‚|j
                  r| j                  |«       ŒE| j                  ||«       ŒX 	 ddd«       t        | j                  «      }t        j                  j                  j                  |«       |dkD  rqt        j                  j                  €J ‚|t        t        j                  j                  «      k(  s.J d|› dt        t        j                  j                  «      › �«       ‚yy# 1 sw Y   Œ¾xY w)	zÊ
        Split nodes into partitions and codegen each partition into separate functions.
        This allows further applying different optimizations (e.g., cudagraph) to
        each function.
        r   r›  Úcudagraph_partitionsz5Each partition must have at least one node but found Nr   zExpect z partition maps but got )r�  r©   r   r8	  r±  r©  r	  r.	  rÐ  ro  rZ   rœ   r:  Úset_all_partition_namesr¨  )rk   rÏ  r­  rÃ  r±  Únum_partitionss         rl   r	  zScheduler._codegen_partitions0  sh  € ð "&×!5Ñ!5Ó!7Ñˆ
�Jäˆz‹?˜QÒÜ�ZÑ Ð!7Ó8¼CÀ
»OÑKÓ8à×,Ñ,¨Z¸ÓDñ 		JÜ(+¨J¸
Ó(Cò JÑ$�	˜9Ü˜9“~¨Ò*ð ØKÌCÐPYËNÐK[Ð\óÐ*ð ×+Ò+Ø—M‘M )Õ,à×3Ñ3°I¸yÕIñJ÷		Jô ˜d×;Ñ;Ó<ˆÜ	�‰×Ñ×4Ñ4°^ÔDð ˜AÒÜ—7‘7×)Ñ)Ð5Ð5Ð5Ø!¤S¬¯©×)?Ñ)?Ó%@Ò@ð Ø˜.Ð)Ð)AÄ#ÄaÇgÁg×F\ÑF\ÓB]ÐA^Ð_óÑ@ð ÷		Jð 		Jús   ÁA&E5Å5E>c                ón  — t         j                  r¿dd l}t        j                  «       }t        «       }t        |«      D ]�  }|j                  dk(  r/|j                  |j                  j                  j                  k(  r nQ|j                  |j                  f}||vs"J d|j                  › d|j                  › d�«       ‚|j                  |«       Œ‘ | j                  | _        | j                   �J ‚| j                  rBt         j"                  j$                  r(t&        j(                  j*                  j-                  «        t&        j(                  j*                  j/                  «        |D �]’  }t0        j3                  t4        j6                  «      r4	 t0        j9                  d|j;                  «       |j=                  «       «       | jA                  |«       t         jB                  rDt&        j(                  j*                  jE                  d„ |jF                  jH                  D «       «       |jK                  «       x}�rs|| j                  k7  s |jM                  «       s|jO                  «       r| jQ                  «        || j                  k7  �r$| j                  rctS        | j                  jT                  «      rD| jV                  �| jY                  «        t&        j(                  j*                  j[                  «        || _        tS        |jT                  «      r™|j\                  €J d	«       ‚d
}	| j_                  «       r5t        | j`                  jc                  «       «      }
|
rte        |
«      d
z   nd
}	t&        j(                  j*                  jg                  |j\                  |	| jh                  «       | j_                  «       r| j                  �| jk                  |«       t&        j(                  j*                  jm                  d„ |jF                  jH                  D «       «       || _7        | jp                  js                  |jt                  «       |jO                  «       rP|jw                  ty        |j{                  «       «      «      \  }}}| j}                  |«      j                  |||«       �n5|jM                  «       r| j�                  |«       �n|jƒ                  «       rxt…        j†                  tˆ        |«      }| j}                  |«      }d
dlEmF} d
dlGmH} d
dlImJ} t—        ||||f«      r|}nt™        dtU        | «      ›�«      ‚|j›                  |«       nŠt—        |tœ        «      r!| j}                  |«      jŸ                  |«       nYt—        |t         t¢        f«      r!| j}                  |«      j¥                  |«       n"t—        |t¦        «      sJ ‚|j©                  «        t         j"                  jª                  r| j}                  |«      j­                  «        | j®                  js                  |j±                  «       «       | j²                  js                  |jµ                  «       «       t—        |t¦        «      sP|jK                  «       }|�>|jT                  dk7  r/| j}                  |«      j·                  «       r| jQ                  «        t¹        d„ |j{                  «       D «       «      r	|| _        �ŒŒd | _        �Œ• | j                  | j                  k7  rU| j                  €J ‚tS        | j                  jT                  «      r(t&        j(                  j*                  j[                  «        d | _        | jQ                  «        y # t>        $ r( t0        j9                  d|j;                  «       «       Y �Œñw xY w)Nr   Ú_compile_innerzDuplicate stack frame ú:zs; did you add a decorator to one of the functions in this stack trace?  If so, try using a context manager instead.z5Generating code for node %s with estimated runtime %fz6Generating code for node %s with estimated runtime 0.0c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   z%Scheduler._codegen.<locals>.<genexpr>‚  s   è ø€ ò DØ!$�C—H•HñDùrµ  r1	  r   c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrw   r³  r´  s     rl   r”   z%Scheduler._codegen.<locals>.<genexpr>³  s   è ø€ ò CØ �—•ñCùrµ  )ÚCUDACombinedSchedulingrr  )ÚXPUCombinedSchedulingztype(self)=rY  c              3  ó<   K  — | ]  }t        |t        «      –— Œ y ­wrw   )rŒ   r�   r  s     rl   r”   z%Scheduler._codegen.<locals>.<genexpr>ï  s   è ø€ ÒJ°A”:˜a¤×/ÑJùrý  )]r(   Ú"check_stack_no_cycles_TESTING_ONLYÚtorch._dynamo.convert_frameÚ	tracebackÚextract_stackr   r«   r²   ÚfilenameÚ_dynamoÚconvert_frameÚ__file__ÚlinenorÜ  rv  rÉ  rs  rÒ   Úautotune_at_compile_timerZ   rœ   r:  Úwrite_get_raw_stream_headerÚregister_alignment_check_inputsr“  rÓ  rÔ  rÆ  r  r  r¦  r’  r‰  Úsize_assertsÚcodegen_deferred_input_assertsr°   r±   rÞ   r"  r   ru  rK   r¶   Úcurrent_stream_idxÚgenerate_stream_ctx_exitr4	  r³   rÂ  r¡  r¹   r¶  r3	  rˆ  Úgenerate_stream_ctx_switchingÚ!codegen_deferred_alignment_copiesrt  r§  r  r`  r×  rÿ   r–   rc  Úcodegen_templater{  r$  rG  rH  ro  Ú codegen.cuda_combined_schedulingrJ	  rv  rs  Ú#codegen.xpu.xpu_combined_schedulingrK	  rŒ   rµ  Úcodegen_combo_kernelr8  Úcodegen_mix_order_reductionrµ   r�   Úcodegen_noderA  rÑ  Údebug_sync_kernelÚcodegen_syncrâ  r  r>  rà   Úready_to_flushr•   )rk   rÒ  rŸ  ÚstackrI  Úframer%  r�   r  Únum_streamsÚunique_streamsrÔ  rÕ  rÖ  Úbackend_rJ	  rs  rK	  rQ  s                      rl   r	  zScheduler._codegenP  s  € Ü×4Ò4Û.ä×+Ñ+Ó-ˆEÜ7A³|ˆDÜ! %›ò �ð —J‘JÐ"2Ò2ØŸ™¨%¯-©-×*EÑ*E×*NÑ*NÒNáØ—~‘~ u§|¡|Ð4�Ø $‘ð Ø,¨U¯^©^Ð,<¸A¸e¿l¹l¸^ð LJð Jó�ð
 —‘˜•ðð #×9Ñ9ˆÔØ×!Ñ!Ð)Ð)Ð)ð ×&Ò&¬6¯=©=×+QÒ+QÜ�G‰G× Ñ ×<Ñ<Ô>ô 	
�‰×Ñ×<Ñ<Ô>àó B	*ˆDÜ×Ñ¤§¡Ô.ð
Ü—I‘IØOØŸ™›Ø×2Ñ2Ó4ôð ×Ñ˜tÔ$ô ×"Ò"Ü—‘×$Ñ$×CÑCñ DØ(,×(8Ñ(8×(>Ñ(>ôDô ð Ÿ™Ó*Ð*ˆvÑ*à˜d×1Ñ1Ò1Ø—~‘~Ô'Ø×'Ñ'Ô)à—J‘J”LØ˜T×0Ñ0Ó0Ø×*Ò*Ô/@Ø×+Ñ+×0Ñ0ô0ð  ×2Ñ2Ð>Ø ×9Ñ9Ô;ÜŸ™×,Ñ,×FÑFÔHØ*0�DÔ'Ü(¨¯©Ô5Ø%Ÿ|™|Ð7ÐVÐ9VÓVÐ7à&'˜Ø×7Ñ7Ô9ä-7¸×8KÑ8K×8RÑ8RÓ8TÓ-U˜Ná;I¤ NÓ 3°aÒ 7Èqð (ô Ÿ™×,Ñ,×GÑGØ"ŸL™LØ'Ø ×;Ñ;ôð ×+Ñ+Ô-°$×2EÑ2EÐ2QØ×2Ñ2°4Ô8ô �G‰G× Ñ ×BÑBñ CØ$(×$4Ñ$4×$:Ñ$:ôCô ð !%ˆDÔØ×%Ñ%×,Ñ,¨T¯_©_Ô=à×ÑÔ!Ø48×4WÑ4WÜ˜Ÿ™Ó)Ó*ó5Ñ1�˜-¨ð × Ñ  Ó(×9Ñ9Ø! 8¨Xöð —‘Ô!Ø×(Ñ(¨Ö.Ø—‘Ô"Ü—{‘{Ô#=¸tÓD�à×+Ñ+¨FÓ3�ÝTÝ8ÝVäØØ#Ð%;Ð=RÐSôð '‘Gä(¨K¬D°«J¨=Ð)9Ó:Ð:Ø×,Ñ,¨TÕ2Ü˜DÔ"9Ô:à× Ñ  Ó(×DÑDÀTÕJÜ˜DÔ#5´}Ð"EÔFà× Ñ  Ó(×5Ñ5°dÕ;ä! $Ô(>Ô?Ð?Ð?Ø—‘”ô �}‰}×.Ò.à× Ñ  Ó(×5Ñ5Ô7à×'Ñ'×.Ñ.¨t×/DÑ/DÓ/FÔGØ×%Ñ%×,Ñ,¨T×-EÑ-EÓ-GÔHä˜dÔ$:Ô;ØŸ™Ó*�àÐ&ØŸ™ vÒ-Ø×(Ñ(¨Ó0×?Ñ?ÔAà—J‘J”LäÑJ¸¿¹Ó9IÔJÔJØ%)�Ö"à%)�Ö"ðEB	*ðH ×Ñ $×"=Ñ"=Ò=ð ×&Ñ&Ð2Ð2Ð2Ü  ×!4Ñ!4×!9Ñ!9Ô:ô —‘×$Ñ$×>Ñ>Ô@à!ˆÔØ�
‰
�øôO !ò Ü—I‘IØPØŸ™›÷ðús   Æ3^Þ-^4Þ3^4c                ó®   — |d   j                  «       }| t        j                  _        || _        |€J ‚| j                  |«      }|j                  ||«      S )rs  r   )rÞ   rZ   rœ   r  rÉ  rc  Úbenchmark_combo_kernel)rk   r±  Únode_benchmark_resultsr  rQ  s        rl   rn	  z Scheduler.benchmark_combo_kernel  sZ   € ð ˜1‘×(Ñ(Ó*ˆØ Œ�‰ÔØ$ˆÔØÐ!Ð!Ð!Ø×"Ñ" 6Ó*ˆØ×-Ñ-¨iÐ9OÓPÐPrn   c                ó  ‡— |}|d   j                  «       Št        ˆfd„|D «       «      sJ d«       ‚t        j                  syddlm} dg }}i }t        |«      D ]›  \  }}|j                  «       }	| j                  |	«      rt        j                  d«       	 | j                  |	«      \  }
}|
|f||<   t        j                  |
«      rt        j                  d|«        y		 ||
z  }|j                  |«       Œ� 	 | j                  ||«      \  }}}||z
  dk  xs |dk  }t        j!                  t"        j$                  «      rP||kD  s|r%t        j                  dt'        ||z  d›«      «       n$t        j                  dt)        ||z  d›«      «       ||z
  |k  xs |S # |$ r.}d
t        |«      v rt        j                  d«       Y d}~ y‚ d}~ww xY w# |$ r-}d
t        |«      v rt        j                  d«       Y d}~y‚ d}~ww xY w)rÀ  r   c              3  óD   •K  — | ]  }|j                  «       ‰k(  –— Œ y ­wrw   r¶  )r’   r�   r  s     €rl   r”   z4Scheduler.speedup_by_combo_kernel.<locals>.<genexpr>  s   øè ø€ ÒK°4�4—?‘?Ó$¨Õ.ÑKùs   ƒ z<All nodes in a combo kernel group must be on the same deviceTrÂ  g        z<ComboKernel: benchmarking may not accurate due to atomic_addz;ComboKernel benchmark: register spilling of %d-th subkernelFrù  zCComboKernel benchmark: return True because of loop-carried variableNg333333Ó?z/can fuse (benchmark): fusing causes %sx speeduprÅ  z3cannot fuse (benchmark): fusing causes %sx slowdown)rÞ   r•   r(   rn	  r  rÃ  rÑ  r–   r²  r  r  rt  rB  rû  r  rÀ   rÓ  rÔ  rÆ  rE   rF   )rk   rÒ  Úsubkernel_nodesrÃ  r  Ú
path1_listro	  rÏ  rø  r±  rº  rà  r  r  Ú	ms2_cloneÚ_path2_listÚsmall_kernelr  s                    @rl   rC  z!Scheduler.speedup_by_combo_kernel  s(  ø€ ð  ˆØ  Ñ#×.Ñ.Ó0ˆäÓK¸?ÔKÔKð 	
ØJó	
ÐKô ×,Ò,Øå;à˜rˆZˆØ!#ÐÜ! /Ó2ò 	$‰HˆAˆuØŸ™Ó)ˆIð ×#Ñ# IÔ.Ü× Ñ ØRôðØ×5Ñ5°iÓ@‘��DØ13°T°
Ð& uÑ-Ü—:‘:˜b”>Ü×$Ñ$ØUØôñ !ð "ð �2‰IˆCØ×Ñ˜dÕ#ð9	$ð<	Ø*.×*EÑ*EØÐ!7ó+Ñ'ˆC�˜Kð ˜Y‘¨Ñ,Ò9°°c±	ˆÜ×"Ñ"¤7§=¡=Ô1Ø�SŠy™LÜ× Ñ ØEÜ #¨¡)¨C Ó2õô
 × Ñ ØIÜ  c¡	¨#˜Ó0ôð
 �Y‰ Ñ$Ò4¨Ð4øðQ $ò à*¬c°!«fÑ4Ü×$Ñ$Ø]ôõ  àûðûð   ò 	à&¬#¨a«&Ñ0Ü× Ñ ØYôô àûð	ús=   ÂAFÃ9G ÆGÆ""GÇ
GÇGÇHÇ"H Ç?H È Hc                óp   — | j                   |   }|j                  €J ‚|j                  j                  «       S rw   )r@  r�   Ú
get_layout)rk   r  rº   s      rl   Úget_buffer_layoutzScheduler.get_buffer_layout_  s5   € Ø×Ñ˜xÑ(ˆØ�x‰xÐ#Ð#Ð#Ø�x‰x×"Ñ"Ó$Ð$rn   c                óØ  — | j                   D ]Û  }|j                  «       sŒ|j                  j                  D ]¯  }t        j
                  j                  j                  |j                  «      }|sŒ9t        |«      dk(  sŒHt        |j                  t        t        f«      rŒi|j                  «       g k(  sŒ}t        j
                  j                  j!                  |j                  «       Œ± ŒÝ y r  )rÒ  rU   r°   r±   rZ   rœ   rÝ  rÈ  r²   r;   rŒ   r'  r?   r>   r¢  Úzero_dim_cpu_tensor_listrÜ  )rk   r�   rH  r]  s       rl   ru  z$Scheduler.update_zero_dim_cpu_tensord  s¯   € Ø—J‘Jò 	HˆDØ�{‰{�}Ø ×,Ñ,×2Ñ2ò 
H�DÜŸW™W×3Ñ3×7Ñ7¸¿	¹	ÓB�FâÜ+¨FÓ3°uÓ<Ü *Ø"ŸM™M¬JÔ8IÐ+Jõ!ð #ŸO™OÓ-°Ó3äŸ™×8Ñ8×<Ñ<¸T¿Y¹YÕGñ
Hñ	Hrn   c                óH   — | j                   �| j                   j                  S y)z:CUDA Stream index that current scheduler node assigned to.N)r©  r¹  rj   s    rl   r[	  zScheduler.current_stream_idxs  s%   € ð ×#Ñ#Ð/Ø×+Ñ+×6Ñ6Ð6àrn   c                ó6   — | j                   x}�t        |«      S y)z9CUDA Stream name that current scheduler node assigned to.N)r[	  r!   )rk   r¹  s     rl   Úcurrent_stream_namezScheduler.current_stream_name{  s#   € ð ×1Ñ1Ð1ˆJÐ>Ü" :Ó.Ð.àrn   c                ó¤   — t        |t        «      rJ ‚| j                  |   }t        j                  j
                  j                  |¬«      | _        y)z6Code-gen to enter the Stream context assigned to node.)r¹  N)rŒ   rA  r¡  rZ   rœ   r:  Úcodegen_cuda_stream_enterr©  )rk   r�   Únode_streams      rl   Úgenerate_stream_ctx_enterz#Scheduler.generate_stream_ctx_enterƒ  sI   € ä˜dÔ$:Ô;Ð;Ð;Ø×)Ñ)¨$Ñ/ˆÜ#$§7¡7×#7Ñ#7×#QÑ#QØ"ð $Ró $
ˆÕ rn   c                ó~   — | j                   €J ‚t        j                  j                  j	                  «        d| _         y)z1Code-gen to exit from the current Stream context.N)r©  rZ   rœ   r:  Úcodegen_cuda_stream_exitrj   s    rl   r\	  z"Scheduler.generate_stream_ctx_exit‹  s2   € à×'Ñ'Ð3Ð3Ð3Ü	�‰×Ñ×5Ñ5Ô7Ø#'ˆÕ rn   c                ó&  — || j                   v sJ ‚t        |t        «      rdn| j                   |   }| j                  |k(  ry| j                  �|€y| j                  €|�| j	                  |«       y| j                  «        | j	                  |«       y)am  Generate stream entering and exiting to properly run node in a multi-stream scenario.

        Stream context switching is only generated if ``node``'s assigned stream is different from
        the previous node's stream. NopKernelSchedulerNodes have stream=None and inherit the
        enclosing stream context (or do nothing if no context is active yet).
        N)r¡  rŒ   rA  r[	  r‚	  r\	  )rk   r�   rF  s      rl   r]	  z'Scheduler.generate_stream_ctx_switching‘  s¢   € ð �t×*Ñ*Ñ*Ð*Ð*ô ˜$Ô 6Ô7ñ à×$Ñ$ TÑ*ð 	ð
 ×"Ñ" fÒ,ð Ø×$Ñ$Ð0°V°^ð Ø×$Ñ$Ð,°Ð1Cà×*Ñ*¨4Õ0ð ×)Ñ)Ô+Ø×*Ñ*¨4Õ0rn   )rÒ  zlist[ir.Operation]r†   rU  )r†   z!dict[str, SchedulerDonatedBuffer]rT  rV  )r  r  r†   r  )r  r  r�   r_   r†   rv   rX  )r  rY  r†   rU  )rÖ  r  r†   rU  )r�   rØ  r†   r_   )rÍ  ú
str | Noner†   rv   r¾  )rø  r_   r†   ræ  )r†   r¿  ©rÒ  rŽ  r†   útuple[float, str]rw   ©rÒ  rŽ  r{  rv   rz  rã  r†   r  )r~  r   r  r6  r†   rˆ	  )r…  úir.MultiTemplateBufferr†   rv   )
r¥  úir.OperationBufferr…  rŠ	  rÏ  r  r�   r�   r†   rU  )r±  rŽ  r†   rv   )rÒ  rŽ  rz  rã  r†   z&tuple[LambdaFuture | None, ModuleType])r�   r_   r‚   r_   r†   rd   )r�   r_   r†   r_   )r�   r_   r‚   r_   rˆ  úOrderedSet[BaseSchedulerNode]r†   r_   )r�   r_   r‚   r_   r  rx   rˆ  rŒ	  )r  ú,dict[BaseSchedulerNode, list[PendingFusion]]rˆ  rŒ	  r†   rU  )
r,  ú1list[tuple[BaseSchedulerNode, BaseSchedulerNode]]r*  ú&dict[BaseSchedulerNode, PendingFusion]r-  r�	  rˆ  rŒ	  rl  rv   )rˆ  rŒ	  r*  r�	  )r6  rŽ	  r7  rŽ	  )rÒ  ræ  rl  rv   r†   ræ  )rX  rã  r†   rU  rN  )rÒ  ræ  rl  rv   r†   rŽ	  r
  )r�   r_   r‚   r_   r  r  r†   rv   )r�   r_   r‚   r_   r“  z!tuple[str, ...] | OrderedSet[str]r†   r  r  r	  )rÒ  r_   rÕ  r_   r  r  r†   rv   )r�   r_   r‚   r_   r†   z,tuple[int, SchedulerNode, sympy.Expr] | None)FT)
r�   r_   r‚   r_   r  rv   r<  rv   r†   rv   )r   r6   r�   r_   r‚   r_   r†   rv   )rH  r3   r  r4   r†   rv   )rH  r3   r  r5   r*  rh  r†   rv   rá  )r¼   r3   rE  rv   r†   r  )TFT)r�   r_   r‚   r_   rE  rv   r1  rv   r<  rv   r†   zint | tuple[int, int, bool])r6  rŽ	  r†   rŽ	  )rÒ  r‡   r†   r   )rz  r_   r†   rU  )r  r6  r†   ÚBaseScheduling)r  rY  r†   r�	  rì  )r²   r  r€  r_  r†   rv   )r�   r_   r†   r†	  )r†   rÁ  )r†   ú6dict[str, ir.IRNode | ir.TorchBindObject | sympy.Expr])r­  úlist[GraphPartitionSignature]r†   rU  )rÃ  r`   rª  r‘	  r†   rÁ  )rÏ  úlist[PartitionType]rÐ  z
list[bool]r†   r’	  )r±  r<   r†   r<   )r†   z9tuple[list[PartitionType], list[GraphPartitionSignature]])rÏ  r“	  r­  r’	  r†   rU  )rÃ  r`   r±  r<   r†   rU  )rÏ  r“	  r­  r’	  r†   z'contextlib.AbstractContextManager[None]©r±  rŽ  r†   z%tuple[float, float, list[str | None]])rÒ  ræ  r†   rv   )r  r  r†   z	ir.Layoutrâ  )r†   r†	  )ory   rz   r{   r  rx  rN  rw  r‰  rÂ  rÅ  rB  ÚpropertyrÉ  Úsetterr¦  r×  rr  r�  r]   r¤  r|  rD  r{  r“  rX  r   r}  r~  r”  rÊ  rœ  rt  rx  r}  rŠ  r‹  rœ  r²  r¾  r  r2  r  r  r$  r0  r4  r<  rm  r’  rô  r?  r  ru  rŠ  rŽ  rœ  r¿  rÆ  rO  r×  rì  rî   r   r5  r  r  rÂ  rI  r:  r;  rQ  rR  r¡  rs  ru  r{  r„  rc  r‰  r~  r“  rI   rŸ  r•  r¦  r´  rÆ  rÚ  rÝ  rð  rŸ  r   r�  r	  r	  r;  r.	  r8	  r2	  r	  r	  rn	  rC  ry	  ru  r[	  r~	  r‚	  r\	  r]	  rS  rT  s   @rl   r  r  þ  s6  ø„ ñó
õo9ób	#ó6
óp(óSó
Qð ò&ó ð&ð ×Ñò(ó ð(ó7ó#ó,ó"óHMPó^KóZ+#óZ ðØ,ðà	 óó6	óSó*ó4#ó&/ó<$ó6!óF	ð8Ø0ð8à	ó8ð, %)ð	à*ðð ðð "ð	ð
 
óð&
>Ø ð
>Ø*6ð
>à	ó
>ð)Ø0ð)à	ó)óVVDðp'8à&ð'8ð +ð'8ð ð	'8ð
 ð'8ð 
ó'8óR
ð OSðØ0ðØAKðà	/óð PØ&ðPØ/@ðPà	óPód>ðà ðð !ðð 3ð	ð
 
óð0à ðð !ðð 'ð	ð
 3óð"D2à$PðD2ð 3ðD2ð 
ó	D2ðLO?àPðO?ð @ðO?ð  Lð	O?ð
 3ðO?ð óO?ðbSà2ðSð @óSð0$àKð$ð $Uó$ð(Cà&ðCð ðCð 
!ó	CôJ3.ój?ð6 à&ð6 ð ð6 ð 
;ó	6 ðp,Ø&ð,Ø/@ð,à	ó,ð\7Ø&ð7Ø/@ð7à	ó7ðr.2Ø&ð.2Ø/@ð.2ØMPð.2à	ó.2ð`$Ø&ð$Ø/@ð$à	ó$ð6<à ð<ð !ð<ð <ð	<ð
 
ó<ð|MØ&ðMØ/@ðMà	óMð^[
Ø&ð[
Ø/@ð[
à	ó[
ózð ;à(ð;ð )ð;ð ð	;ð
 
ó;ðz`Ø&ð`Ø/@ð`à	5ó`ðL "Ø*.ðzMà ðzMð !ðzMð ð	zMð
 $(ðzMð 
ózMðx;Ø&ð;Ø/@ð;à	ó;ðz3)Øð3)Ø(9ð3)ØBSð3)à	ó3)ór%ðTØðØ 'ðØ7Jðà	óô;ð !Ø.3Ø*.ðmKà ðmKð !ðmKð ð	mKð
 (,ðmKð $(ðmKð 
%ómKð^dà ðdð !ðdð 
ó	dðLR
Ø&ðR
Ø/@ðR
à	óR
ðh6Ø Qð6à	:ó6ð@4Ø@ð4à	ó4ó	8ó*ó4ð
à)ðð 
óó&'ó*%ó5ð$
Øð
Ø+:ð
à	ó
ó=ð~ ò!ó ð!ðFØ%ðà	óð"à	?óð 'à1ð'ð 
ó'ðRBHà ðBHð LðBHð 
"ó	BHðHK Ø-ðK Ø@JðK à	&óK ðZ"
Ø0ð"
à	 ó"
ðH?à&ð?ð 
!ó?ðBà&ðð 
!óð>%Ø,ð%à	 ó%ð@5&à	Bó5&ðn";à'ð";ð 2ð";ð 
ó	";óH>ó0ð)
à ð)
ð +ð)
ð 
ó	)
ðVØ-ðØ;Xðà	0óð6-AØ-ð-AØ;Xð-Aà	ó-Aó^ó@oðbQØ4ðQà	.óQóN5ó`%ó
Hð òó ðð òó ðó
ó(÷1rn   c                  ó<  ‡ — e Zd Zdˆ fd„Zdd„Zdd„Z	 	 	 	 	 	 dd„Z	 	 	 	 	 	 dd„Z	 	 	 	 	 	 dd„Z	 	 	 	 	 	 dd„Z		 	 	 	 dd„Z
	 	 	 	 	 	 	 	 dd	„Z	 d	 	 	 	 	 	 	 dd
„Zdd„Zdd„Zdd„Zd d„Zdd„Z	 	 	 	 d!d„Zd"d„Z	 	 	 	 	 	 d#d„Z	 	 	 	 d$d„Z	 d	 	 	 	 	 d%d„Zˆ xZS )&r�	  c                ó0   •— t         ‰| �  «        || _        y rw   )r>  rx  r  )rk   r  rG  s     €rl   rx  zBaseScheduling.__init__°  s   ø€ Ü‰ÑÔØ"ˆ�rn   c                óR   — | j                   r| j                   j                  «        y y rw   )r  rs  rj   s    rl   Úfree_buffers_in_schedulerz(BaseScheduling.free_buffers_in_scheduler´  s   € Ø�>Š>Ø�N‰N×'Ñ'Õ)ð rn   c                ó   — t        «       S )z0Return a set of .codegen.common.BackendFeature()r   r  s     rl   Úget_backend_featuresz#BaseScheduling.get_backend_features¸  s
   € ä‹|Ðrn   c                ó   — t         ‚)zO
        Check whether node1 and node2 can be vertically fused or not.
        r*  r  s      rl   r   z BaseScheduling.can_fuse_vertical¼  ó
   € ô "Ð!rn   c                ó   — t         ‚)zQ
        Check whether node1 and node2 can be horizontally fused or not.
        r*  r  s      rl   r  z"BaseScheduling.can_fuse_horizontalÄ  rž	  rn   c                óâ  — |j                  «       }t        |t        j                  «      sy|j	                  «       syt        |j
                  t        j                  «      r�t        |j
                  j                  «      dk(  xrk t        |j
                  j                  d   t        j                  «      xr8 |j
                  j                  d   j                  «       |j                  «       k(  S y)av  
        A Multi-Output Template (referenced in #144012) is a template node
        with MultiOutputLayout, and its output buffers are instances of MultiOutput.
        In this context, we verify whether node1 represents the Multi-Output Template
        and node2 corresponds to one of its outputs. If so, we further check if
        backend supports this fusion.

        Fr   r   )rÆ  rŒ   r+   ra  rV   r�   r=   r©   ró  rª  r  )rk   r�   r‚   rÖ  s       rl   rø  z.BaseScheduling.can_fuse_multi_outputs_templateÌ  s¹   € ð ×.Ñ.Ó0ˆÜ˜,¬×(9Ñ(9Ô:ØØ×5Ñ5Ô7Øä�e—j‘j¤"§.¡.Ô1ä�E—J‘J×%Ñ%Ó&¨!Ñ+ò OÜ˜uŸz™z×0Ñ0°Ñ3´R·Y±YÓ?òOà—J‘J×%Ñ% aÑ(×1Ñ1Ó3°|×7LÑ7LÓ7NÑNðð rn   c                óØ  — |j                  «       s|j                  «       rt        j                  ||«      S t        j	                  ||«      rt        ||«      S t        |t
        «      r|j                  |«      S t        |t        «      rLt        |t        «      r<t        |j                  t        j                  «      sJ ‚t        j                  ||«      S t        j                  ||«      S )z 
        Fuse two nodes
        )r$  ro  rr   r‰   rð   r8  rŒ   rT  r�  r�   r�   r+   r@  rV  r_  rµ   r  s      rl   rr   zBaseScheduling.fuseæ  s¾   € ð ×ÑÔ ×!1Ñ!1Ô!3Ü-×2Ñ2°5¸%Ó@Ð@Ü×7Ñ7¸¸uÔEÜ*¨5°%Ó8Ð8Ü˜Ô6Ô7Ø—?‘? 5Ó)Ð)Ü˜Ô8Ô9¼jØ”=ô?
ô ˜eŸj™j¬"×*DÑ*DÔEÐEÐEÜ7×EÑEÀeÈUÓSÐSä%×*Ñ*¨5°%Ó8Ð8rn   c                ó   — t         ‚)z[
        Process the iteration sizes in case a transformation needs to be applied.
        r*  )rk   r~  s     rl   rd  zBaseScheduling.group_fnú  rž	  rn   c                ó   — t         ‚)zã
        Given a template node, generate a kernel.

        This function is only available for triton now. If the third-party backend behaves as a sub-class
        of TritonScheduling, it can override it or reuse it.
        r*  )rk   rÕ  Úepilogue_nodesr÷  s       rl   r_	  zBaseScheduling.codegen_template  s
   € ô "Ð!rn   c                ó   — t         ‚©zD
        Generate a kernel given a list of pre-fused nodes.
        r*  )rk   rÒ  r{  rz  s       rl   rx  z.BaseScheduling.generate_kernel_code_from_nodes  s
   € ô "Ð!rn   c                ó   — t         ‚r¦	  r*  r%  s     rl   rd	  zBaseScheduling.codegen_node  ó
   € ô "Ð!rn   c                ó   — t         ‚rw   r*  r%  s     rl   rc	  z*BaseScheduling.codegen_mix_order_reduction!  r,  rn   c                ó   — t         ‚)zt
        Generate synchronization code for the kernel. This method depends on the hardware characteristics.
        r*  rj   s    rl   rf	  zBaseScheduling.codegen_sync$  r¨	  rn   c                 ó   — y)z•
        Check whether the backend is requesting the scheduler to flush the generated kernel.
        If not supported, please return False.
        Fr~   rj   s    rl   rg	  zBaseScheduling.ready_to_flush*  s   € ð
 rn   c                ó   — t         ‚)z]
        Flush the generated kernel and python wrapper code to the source code file.
        r*  rj   s    rl   ru  zBaseScheduling.flush1  r¨	  rn   c                ó   — t         ‚)rs  r*  rO  s     rl   rt  z$BaseScheduling.benchmark_fused_nodes7  ó
   € ô "Ð!rn   c                ó   — t         ‚)z�
        Benchmark a compiled module and return the execution time
        in milliseconds on randomly generated inputs.
        r*  )rk   r~  s     rl   r}  z)BaseScheduling.benchmark_codegened_module@  s
   € ô
 "Ð!rn   c                 ó   — y)zŒ
        Return an unsigned integer which represents the priority of this fusion pair.
        The smaller is with higher priority.
        r   r~   r  s      rl   rg  z'BaseScheduling.get_fusion_pair_priorityG  s   € ð rn   c                ó   — t         ‚)z¡
        Benchmark the list of nodes to combine and return the execution time
        and memory copy time in milliseconds on randomly generated inputs.
        r*  )rk   r±  ro	  s      rl   rn	  z%BaseScheduling.benchmark_combo_kernelP  r®	  rn   c                ó|   — |r:ddl m}  |||«      }t        j                  j                  j                  ||«       y y )Nr   )Ú'set_kernel_post_grad_provenance_tracing)r€  r³	  rZ   rœ   r:  Úwrite_provenance_debug_handle)rk   Únode_schedulerµ  r³	  Údebug_handles        rl   Úcodegen_commentzBaseScheduling.codegen_commentY  s>   € ñ
 ÝUáBØØóˆLô �G‰G× Ñ ×>Ñ>Ø˜\õð rn   )r  zScheduler | NonerT  )r  r6  r†   zOrderedSet[BackendFeature]r
  r4  )r~  rQ  r†   z"tuple[tuple[sympy.Expr, ...], ...])rÕ  r_   r¤	  rŽ  r÷  rŽ  r†   r†	  rw   r‰	  )r�   z"FusedSchedulerNode | SchedulerNoder†   rU  )r�   r8  r†   rU  rV  r‡	  )r~  r   r†   rˆ	  r  r”	  )rµ	  rŽ  rµ  r†	  r†   rU  )ry   rz   r{   rx  rš	  rœ	  r   r  rø  rr   rd  r_	  rx  rd	  rc	  rf	  rg	  ru  rt  r}  rg  rn	  r·	  rS  rT  s   @rl   r�	  r�	  ¯  s�  ø„ õ#ó*óð"Ø&ð"Ø/@ð"à	ó"ð"Ø&ð"Ø/@ð"à	ó"ðØ&ðØ/@ðà	óð49Ø&ð9Ø/@ð9à	ó9ð("Ø3ð"à	+ó"ð"à(ð"ð 4ð"ð 4ð	"ð
 
ó"ð$ %)ð		"à*ð	"ð ð	"ð "ð		"ð
 
ó	"ó"ó"ó"óó"ð"Ø0ð"à	ó"ó"ðØ&ðØ/@ðà	óð"Ø4ð"à	.ó"ð #'ðà2ðð  ðð 
÷	rn   r�	  )r†   z$torch._inductor.codecache.LocalCache)rø  r_   r†   r  )rø  r_   r†   zCallable[[Any], Any] | None)rø  r_   r†   rm  )r,  r   r†   r  )r�   r_   ró  rÞ  r@  rk  r†   rU  )rÝ  ú)FusedSchedulerNode | GroupedSchedulerNoder†   rU  )rÝ  r¸	  r  r  r·   ræ  r†   rU  )r~   )rè  zlist[list[int]]r~  rÙ  rî  r×  r†   z	list[int])r÷  rŠ	  r‰  r‹	  r†   rU  rä  )r
  r  r  r  r  r  r  r  r  rC   r†   ztuple[int, int])r  r¯  r  r¯  r
  r  r  r  r  r  r  r  r  rC   r†   rv   rV  )r�   z	ir.IRNoder†   rÁ  )r�   r_   r†   rÁ  )rÕ  r_   r†   rv   )r�   r_   r‚   r_   )ÃÚ
__future__r   r•  r6	  rZ  rê  r�  rÔ  rÔ  rB  r  rÐ  r*  r+  rO	  rG  r   r   Úconcurrent.futuresr   r   r   r	   r
   r   r   Útyping_extensionsr   Útorch.utils._ordered_setr   r+   r   r   Úcollections.abcr   r   r   Útypesr   Útorch._inductor.codegen.wrapperr   r7  r   rä   rŸ  Útorch._inductor.async_compileÚtorch.utils._pytreer  Ú_pytreerö  Útorch._dynamo.utilsr   r   Ú torch._inductor.autotune_processr   Útorch._inductor.codecacher   r   Útorch._inductor.irr   Útorch._inductor.metricsr   r    Útorch._inductor.stream_utilsr!   Ú%torch.fx.experimental.symbolic_shapesr"   Útorch.utils._sympy.symbolr#   r$   r%   Útorch.utils._tritonr&   r—  r'   r(   r)   r*   r,   Úanalyze_preserves_zero_maskr-   Úcodegen.commonr.   r/   r0   Úcomm_analysisr1   r2   r3   r4   r5   r6   Úexcr7   r8   Úfx_utilsr9   r:   r;   r<   r=   r>   r?   r  r@   r”  rA   rB   Úruntime.hintsrC   rD   Úruntime.runtime_utilsrE   rF   r�   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   ÚvirtualizedrZ   Ú	getLoggerry   r“  Ú_loggingÚgetArtifactLoggerr  r™  r  r	  rÿ   r`   r|   ra   rb   Ú	dataclassrd   r€   r‰   r  r]  r_   r¸  r­  r¬  r  r³  r  r)  rG  rò  r�  rA  r�   râ  rí  rµ   r8  rV  ro  rì  rñ  rý  r  r  r  r]  rn  rm  r'  r-  r2  r=  rB  rD  rG  rI  rK  r  r�	  r~   rn   rl   ú<module>rØ	     sÈ  ðÞ "ã Û Û Û Û Û Û Û Û Û 	Û Û Û Û ß ,ß 3ß BÕ BÝ 'å /ç )ñ ß<Ñ<Ý åJå5ã ã Û $ß $Ð $ß 6Ý Eß ?Ý 7ß MÝ 8Ý >ß OÑ OÝ *ç D× DÝ Dß MÑ M÷÷ ;Ó :ß 2Ý $÷÷ õ  ß Jß :ß 7Ý &÷÷ ÷ ÷ ÷ õ( ð €g×Ò˜Ó!€Ø�^Š^×-Ò-¨h¸ÓA€
Ø—N’N×4Ò4°X¸ÓOÐ Ø Ÿ>š>×;Ò;ØÐ$óÐ ð —’×1Ò1°(¸LÓI€àÐ 3Ñ4€ˆyÓ 4ÙˆTƒ]€Ùˆtƒ_€ð ×Ò÷Dð Dó ðDð* ×Ò÷(ð (ó ð(÷Yñ Yðx ×Ò÷h8ð h8ó ðh8ðV ×Òô1˜_ó 1ó ð1÷H1ñ H1ðV ‡‚ò2ó ð2óó,'ó #ðL €×Ò˜TÔ"÷
ð 
ó #ð
ó*÷ñ ð *KØ
ð*Kà4ð*Kð ,ð*Kð 
ó	*KôZ"*Ð 1ô "*ôJ5Ð.ô 5ôS*Ð%ô S*ðl
Ø:ðà	óð$Ø:ðàðð $ðð 
ó	ô8k*Ð*ô k*ô\[GÐ0ô [Gô|:-Ð+=ô :-ôzC:Ð!3ô C:ôLbÐ,ô bðP #%ð+Ø#ð+àð+ð  ð+ð ó	+ð\0Ø%ð0Ø1Cð0à	ó0ó8
1ð(Øð(àð(ð ð(ð ð	(ð
 #ð(ð ó(ð6Ø	ðà	ðð ðð ð	ð
 ðð ðð #ðð 
óð@ ×Ò÷
ð 
ó ð
ð> +˜9Ÿ?š?Ó,Ð ó4óð$Ø
ðàóó&"ó"óóóPó@÷nJ1ñ nJ1÷bUxò xrn   