Ë
    L²jÒ'  ã                   ó2  — d Z ddlZddlmZ ddlmZmZ ddlmZ ddl	Z	ddl
Z	ddlmZ ddlmZ ddlmZ dd	lmZmZmZmZmZmZ dd
lmZmZmZmZmZ ddlmZ ddl m!Z! de	jD                  jF                  de$e%   fd„Z&de	jD                  jN                  de(e	jR                  e	jD                  jT                  f   fd„Z+de	jD                  jN                  de%de,dz  fd„Z-de	jD                  jN                  de%de,dz  fd„Z.de	jD                  jN                  de%fd„Z/de	jD                  jN                  de0e,dz     fd„Z1de	jD                  jN                  dee   defd„Z2 G d„ d«      Z3 e!d e3«       ¬«       	 	 d&d ed!ef   d"ee   d#e4d$e4ded!ee   f   f
d%„Z5y)'a�  
This module implements CUDA graphs support for TorchDynamo backends.

CUDA graphs allow for capturing and replaying GPU operations, which can significantly
reduce CPU overhead in GPU-accelerated PyTorch models. This module provides:

- CUDA graph creation and management for both forward and backward passes
- Input mutation detection and handling
- Device compatibility checking
- Stack trace management for debugging
- Integration with TorchInductor's cudagraph trees

The backend supports two main modes:
1. cudagraphs: Full CUDA graph support with both forward and backward pass optimization
2. cudagraphs_inner: Lower-level CUDA graph implementation used for benchmarking

Key components:
- CudagraphsBackend: Main backend class for CUDA graph integration
- Mutation detection utilities to ensure graph safety
- Device mapping and compatibility checks
- Stack trace collection for debugging
é    N)Údefaultdict)ÚCallableÚSequence)ÚAny)Úconfig)Úaot_autograd)Ú	boxed_nop)ÚBoxedDeviceIndexÚ'check_multiple_devices_or_any_cpu_nodesÚformat_default_skip_messageÚget_mutation_stack_traceÚget_placeholder_infoÚ#log_cudagraph_skip_and_bump_counter)Ú	BoxedBoolÚcount_tangentsÚ%get_first_incompatible_cudagraph_nodeÚnum_fw_fixed_argumentsÚoutput_node)ÚStorageWeakRefé   )Úregister_backendÚgÚreturnc           	      ó¨  — dt         t        t        f   dt        fd„}t        t        «      }d}t	        «       }| j
                  D �]‰  }|j                  dk(  rkt         ||j                  «      t        j                  «      r;|t         ||j                  «      j                  «       «         j                  |«       |dz  }Œ~|j                  dk(  sŒŽt        |j                  d«      sŒ¥|j                  j                   }t#        |j$                  «      D ]·  \  }}|t'        |j(                  «      k  r|j(                  |   }	n2|j*                  |j,                  vrŒG|j,                  |j*                     }	d	}
|j.                  r|j.                  j0                  rd
}
|
sŒ‰||t         ||	j                  «      j                  «       «         z  }Œ¹ �ŒŒ |S )NÚmetar   c                 ó   — d| v r| d   S | d   S )NÚvalÚfake_result© )r   s    ú`/var/www/html/ia-enhanced/venv/lib/python3.12/site-packages/torch/_dynamo/backends/cudagraphs.pyÚmeta_fkz%find_input_mutations.<locals>.meta_fk7   s   € Ø# t™mˆt�E‰{ÐD°°mÑ1DÐDó    r   Úplaceholderr   Úcall_functionÚ_schemaFT)ÚdictÚstrr   r   ÚsetÚnodesÚopÚ
isinstancer   ÚtorchÚTensorr   Ú_typed_storageÚaddÚhasattrÚtargetr%   Ú	enumerateÚ	argumentsÚlenÚargsÚnameÚkwargsÚ
alias_infoÚis_write)r   r!   ÚinputsÚ	input_idxÚmutated_inputsÚnÚschemaÚiÚargÚargumentÚmut_args              r    Úfind_input_mutationsrC   6   s€  € ðE”dœ3¤˜8‘nð E¬ó Eô œÓ€FØ€IÜ“U€NØ�W‰Wó ˆØ�4‰4�=Ò Ü™' !§&¡&›/¬5¯<©<Ô8Ø”~¡g¨a¯f©f£o×&DÑ&DÓ&FÓGÑH×LÑLÈYÔWØ˜‰N‰IØ�T‰T�_Ó$Ü˜1Ÿ8™8 YÔ/Øà—X‘X×%Ñ%ˆFÜ# F×$4Ñ$4Ó5ò ‘��3Ø”s˜1Ÿ6™6“{’?Ø Ÿv™v a™y‘Hà—x‘x q§x¡xÑ/Ø Ø Ÿx™x¨¯©Ñ1�HØ�Ø—>’>Ø—~‘~×.Ò.Ø"&˜Úð # fÜ&¡w¨x¯}©}Ó'=×'LÑ'LÓ'NÓOñ'ñ ‘Nòðð: Ðr"   Úgmc                 óê   — i }| j                   j                  D ]W  }|j                  j                  dd «      }t	        |t
        j                  «      sŒ:|j                  |vsŒI|||j                  <   ŒY |S )Nr   )Úgraphr)   r   Úgetr+   r,   r-   Údevice)rD   Údevice_node_mappingr=   Úts       r    Úget_device_node_mappingrK   ]   sg   € ð >@ÐØ�X‰X�^‰^ò .ˆØ�F‰F�J‰J�u˜dÓ#ˆÜ�aœŸ™Õ&¨1¯8©8Ð;NÒ+NØ,-Ð §¡Ò)ð.ð Ðr"   Ú	aot_modelÚ	num_fixedc                 óž   — t        | j                  «      t        t        |«      «      z
  }|sy t	        | j                  «      }t        ||«      S ©N)rC   rF   r(   Úranger   r   )rL   rM   Úmutation_indicesÚplaceholderss       r    Ú3check_for_mutation_ignore_cuda_graph_managed_tensorrS   h   sD   € ô ,¨I¯O©OÓ<¼sÄ5ÈÓCSÓ?TÑTÐÙØä'¨	¯©Ó8€LÜ# LÐ2BÓCÐCr"   c                 óÀ   — t         j                  st        | |«      x}r|S t        t	        | «      «      x}r|S t        | «      x}rt        d|j                  › d�«      S y )Nzincompatible op (ú))r   Ú(cudagraph_backend_support_input_mutationrS   r   rK   r   r   r6   )rL   rM   Úmut_skipÚskipÚnodes        r    Úcheck_for_skiprZ   s   sz   € Ü×:Ò:ÜJØ�yó
ð 
ˆ8ð 
ð ˆOä6Ü 	Ó*óð €tð ð ˆä4°YÓ?Ð?€tÐ?Ü*Ð->¸t¿y¹y¸kÈÐ+KÓLÐLàr"   c                 óv   — t        t        t        | «      «      «      }|j                  dk(  sJ ‚|j                  S )NÚcuda)ÚnextÚiterrK   ÚtypeÚindex)rD   rH   s     r    Úget_device_indexra   …   s3   € Ü”$Ô.¨rÓ2Ó3Ó4€FØ�;‰;˜&Ò Ð Ð Ø�<‰<Ðr"   c                 ó$  — t        | «      }t        |j                  «      dk(  sJ ‚|j                  d   }t        |d«      sg S |D �cg c]>  }t	        |t
        j                  j                  j                  «      r|j                  nd ‘Œ@ c}S c c}w )Nr   r   Ú__iter__)
r   r4   r5   r0   r+   r,   ÚfxrY   ÚNodeÚstack_trace)rD   Úoutputr5   r@   s       r    Úget_stack_tracesrh   ‹   s€   € Ü˜‹_€FÜˆv�{‰{Ó˜qÒ Ð Ð Ø�;‰;�q‰>€DÜ�4˜Ô$Øˆ	ð öàô ' s¬E¯H©H¯M©M×,>Ñ,>Ô?ˆ�ŠÀTÑ	Iòð ùò s   ÁABÚdynamo_modelÚdynamo_inputsc           	      óÆ  ‡‡‡‡— ddl mŠ t        d«      Št        d «      Š	 ddt        j
                  j                  dt        t           dt        dt        fˆˆˆˆfd„}dt        j
                  j                  dt        t           dt        fˆˆˆfd	„}t        ||t        j                  |d¬
«      t        j                  j                  j                  ¬«      } || ‰«      S )Nr   )Úcudagraphify_implTrL   Ú
aot_inputsÚis_inferencer   c                 óª  •— t        | |«      }t        t        ‰
«      t        |«      «      }t        | |«      x}r%t	        j
                  ‰	«       t        d|› �«       |S ‰j                  t        | «      «        ‰||t        |«      ‰j                  d|t        | «      t        | j                  «      t        | j                  «      ¬«	      }d|_        |S )Núskipping cudagraphs due to F©Údevice_indexÚis_backwardrn   Ústack_tracesrR   Úmutated_input_idxsT)r	   r   r4   rZ   r   Údisabler   r(   ra   rP   Úvaluerh   r   rF   rC   Ú_boxed_call)rL   rm   rn   ÚinterpÚfixedÚskip_msgÚoutÚboxed_device_indexrl   Údo_cudagraphsrj   s          €€€€r    Úforward_cudagraphsz&cudagraphs.<locals>.forward_cudagraphs�   sÇ   ø€ ô
 ˜9 jÓ1ˆÜ&¤s¨=Ó'9¼3¸z»?ÓKˆÜ% i°Ó7Ð7ˆ8Ð7Ü×Ñ˜mÔ,Ü/Ø-¨h¨ZÐ8ôð ˆMà×ÑÔ/°	Ó:Ô;ÙØØÜ�%‹LØ+×1Ñ1ØØ%Ü)¨)Ó4Ü-¨i¯o©oÓ>Ü3°I·O±OÓDô

ˆð ˆŒØˆ
r"   c                 óð  •‡ ‡— t        ‰ |«      }‰s‰ S t        ‰ «      }t        ‰ |«      x}rpt        d|› �«       ‰	j                  }|€d}t
        j                  j                  j                  |d¬«      Š‰€J ‚dt        t           dt        fˆ ˆfd„}d|_        |S  ‰
||t        |«      t        ‰ «      ddt        ‰ «      t        ‰ j                   «      t#        ‰ j                   «      ¬	«	      }d|_        |S )
Nrp   r   F)Úcreate_if_none_existsr:   r   c                 ó4   •— ‰j                  «         ‰| «      S rO   )Úset_to_running_backward)r:   rL   Úmanagers    €€r    Úfnz3cudagraphs.<locals>.backward_cudagraphs.<locals>.fnÐ   s   ø€ à×/Ñ/Ô1Ù  Ó(Ð(r"   Trq   )r	   r   rZ   r   rw   r,   Ú	_inductorÚcudagraph_treesÚget_managerÚlistr   rx   rP   ra   rh   r   rF   rC   )rL   rm   ry   rz   r{   Ú
device_idxr…   r|   r„   r}   rl   r~   s   `       @€€€r    Úbackward_cudagraphsz'cudagraphs.<locals>.backward_cudagraphsº   s  ú€ ô ˜9 jÓ1ˆÙØÐä˜yÓ)ˆÜ% i°Ó7Ð7ˆ8Ð7Ü/Ø-¨h¨ZÐ8ôð
 ,×1Ñ1ˆJØÐ!Ø�
Ü—o‘o×5Ñ5×AÑAØ°%ð Bó ˆGð Ð&Ð&Ð&ð)œ4¤™9ð )¬ö )ð
 "ˆBŒNØˆIáØØÜ�%‹LÜ)¨)Ó4ØØÜ)¨)Ó4Ü-¨i¯o©oÓ>Ü3°I·O±OÓDô

ˆð ˆŒØˆ
r"   )rn   )Úfw_compilerÚbw_compilerÚinference_compilerÚkeep_inference_input_mutations)F)Útorch._inductor.cudagraph_treesrl   r   r
   r,   rd   ÚGraphModuler‰   r   Úboolr   Ú	functoolsÚpartialÚ_dynamor   Ú%cudagraph_backend_keep_input_mutation)ri   rj   r   r‹   Úaot_cudagraphsr}   rl   r~   s    `   @@@r    Ú
cudagraphsr˜   —   sÌ   û€ ÝAä˜d“O€MÜ)¨$Ó/Ðð
 #ñÜ—8‘8×'Ñ'ðäœ‘Iðô ðô 
÷	ð ð:*Ü—8‘8×'Ñ'ð*Ü59¼#±Yð*ä	÷*ôX "Ø&Ø'Ü$×,Ñ,Ð-?ÈdÔSÜ',§}¡}×';Ñ';×'aÑ'aô	€Nñ ˜,¨Ó6Ð6r"   c                   ón   — e Zd ZdZedd„«       Zedej                  j                  de	e
   de
fd„«       Zy)	ÚCudagraphsBackendr˜   r   Nc                  ó   — ddl m}   | «        y )Nr   ©Úreset_cudagraph_trees)r�   r�   rœ   s    r    ÚresetzCudagraphsBackend.resetò   s   € åIáÕr"   Úmodelr:   c                 ó   — t        | |«      S rO   )r˜   )rŸ   r:   s     r    Ú__call__zCudagraphsBackend.__call__ø   s   € ä˜% Ó(Ð(r"   )r   N)Ú__name__Ú
__module__Ú__qualname__Úcompiler_nameÚstaticmethodrž   r,   rd   r‘   r   r   r¡   r   r"   r    rš   rš   ï   sP   „ Ø €Màò ó ð ð
 ð)˜Ÿ™×,Ñ,ð )°h¸s±mð )Èò )ó ñ)r"   rš   r˜   )r6   Úcompiler_fnrŸ   .r:   Úcopy_outputsÚcopy_inputsc                 óÄ  ‡‡‡‡‡	— t        |t        t        f«      sJ ‚‰r$|D �cg c]  }t        j                  |«      ‘Œ c}Šnt        |«      Št        j
                  j                  «        t        j
                  j                  «       }|j                  t        j
                  j                  «       «       t        j
                  j                  |«      5   | |Ž  ddd«       |j                  «        t        j
                  j                  «       j                  |«       t        j
                  j                  «        t        j
                  j                  «       Št        j
                  j                  ‰|¬«      5   | ‰Ž Š	ddd«       t        ‰	t        t        f«      s‰	fŠ	dt        dt        t           fˆˆˆˆˆ	fd„}|S c c}w # 1 sw Y   ŒîxY w# 1 sw Y   ŒRxY w)zBThis isn't registered as a backend, but is used in some benchmarksN)ÚstreamÚ
new_inputsr   c                  óò   •— t        ‰«      t        | «      k(  sJ ‚‰r%t        ‰| «      D ]  \  }}|j                  |«       Œ ‰j                  «        ‰r‰D �cg c]  }|j	                  «       ‘Œ c}S ‰S c c}w rO   )r4   ÚzipÚcopy_ÚreplayÚclone)	r¬   ÚdstÚsrcÚxr©   r¨   rF   Ústatic_inputsÚstatic_outputss	       €€€€€r    Úrunzcudagraphs_inner.<locals>.run   sp   ø€ Ü�=Ó!¤S¨£_Ò4Ð4Ð4ÙÜ ¨zÓ:ò ‘��SØ—	‘	˜#•ðà�‰ŒÙØ'5Ö6 !�A—G‘G•IÒ6Ð6à!Ð!ùò 7s   ÁA4)r+   r‰   Útupler,   Ú
zeros_liker\   ÚsynchronizeÚStreamÚwait_streamÚcurrent_streamr«   Ú	CUDAGraphrF   r   r   )
rŸ   r:   r¨   r©   r´   r«   r·   rF   rµ   r¶   s
     ``   @@@r    Úcudagraphs_innerr¿     sm  ü€ ô �fœt¤U˜mÔ,Ð,Ð,ÙØ6<Ö=°œ×)Ñ)¨!Õ,Ò=‰ä˜V›ˆô 
‡J�J×ÑÔÜ�Z‰Z×ÑÓ €FØ
×Ñ”u—z‘z×0Ñ0Ó2Ô3Ü	�‰×	Ñ	˜6Ó	"ñ Ùˆv‰÷à
×ÑÔÜ	‡J�J×ÑÓ×+Ñ+¨FÔ3Ü	‡J�J×ÑÔô �J‰J× Ñ Ó"€EÜ	�‰×	Ñ	˜%¨Ð	Ó	/ñ /Ù Ð.ˆ÷/ä�n¤t¬U mÔ4Ø(Ð*ˆð	"œð 	"¤¬#¡÷ 	"ñ 	"ð €JùòA >÷ð ú÷/ð /ús   ¤GÃG
Å?GÇ
GÇG)TT)6Ú__doc__r“   Úcollectionsr   Úcollections.abcr   r   Útypingr   r,   Útorch.fxÚtorch._dynamor   Útorch._dynamo.backends.commonr   Ú torch._dynamo.backends.debuggingr	   Útorch._inductor.cudagraph_utilsr
   r   r   r   r   r   Útorch._inductor.utilsr   r   r   r   r   Ú torch.multiprocessing.reductionsr   Úregistryr   rd   ÚGraphr(   ÚintrC   r‘   r&   rH   re   rK   r'   rS   rZ   ra   r‰   rh   r˜   rš   r’   r¿   r   r"   r    ú<module>rÎ      sÐ  ðñó. Ý #ß .Ý ã Û Ý  Ý 6Ý 6÷÷ ÷õ õ <å &ð$˜EŸH™HŸN™Nð $¨s°3©xó $ðNØ�‰×Ñðà	ˆ%�,‰,˜Ÿ™Ÿ™Ð
%Ñ&óðDØ�x‰x×#Ñ#ðDØ03ðDàˆ4�ZóDð˜eŸh™h×2Ñ2ð ¸sð ÀsÈTÁzó ð$˜Ÿ™×-Ñ-ð °#ó ð	˜Ÿ™×-Ñ-ð 	°$°s¸T±zÑ2Bó 	ðU7˜UŸX™X×1Ñ1ð U7À(È3Á-ð U7ÐTWó U7÷p)ñ )ñ  �lÑ0AÓ0CÕ Dð Øñ	)Ø�C˜�HÑð)à�S‰Mð)ð ð)ð ð	)ð
 ˆc�8˜C‘=Ð Ñ!ô)r"   