LjGUddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZddlmZddlmZmZddlmZmZddlZddlZddlZddlmZddlmcm Z!ddl"m#Z#m$Z$ddl%m&Z&dd l'm(Z(dd l)m*Z+dd l,m-Z-m.Z.dd l/m0Z0dd l1m2Z2ddl3m4Z4m5Z5ddl6m7Z7ddl8m9Z9ddl:m;Z;ddlZ>ddl?m@Z@mAZAddlBmCZCmDZDmEZEmFZFmGZGmHZHddlImJZJddlKmLZLddlMmNZNdd lOm*Z*ddlPmQZQddlRmSZSmTZTmUZUmVZVddlWmXZXddlYmZZZm[Z[m\Z\ddl]m^Z^ddl_m`Z`ddlambZbdd lcmdZdmeZemfZfmgZgmhZhdd!limjZjmkZkmlZlerddlmZnddloZoe*jZqd"erd#<ejetZud$erd%<ejjZwejjZxeGd&d'ZyeGd(d)ZzeGd*d+Z{dqd,Z|drd-Z}drd.Z~dsd/ZGd0d1Zdqd2ZeZdtd3Zdud4Z dvd5Z dw dxd7Zdqd8Zdqd9Zdqd:Zdqd;Zdqd<Zdqd=Zdqd>Zdqd?Z dyd@ZdzdAZ d{dBZ d| d}dCZ d~dDZddEZddFZddGZddHZddIZdddJZddKZ d ddLZ d ddMZ ddd6d6dN ddOZdddP ddQZedRZddSZdsdTZddUZejHddVZ ddWZddXZ ddYZ ddZZdd[Zdd\Zdd]Zdqd^Z dd_Z d dd`Z ddaZddbZ ddcZdddZddeZ ddfZddglmZddhZddiZ d ddjZ ddkZ ddlZ ddmZ dddn ddoZ d ddpZy)) annotationsN) defaultdictdeque)Callable) dataclassreplace)Any TYPE_CHECKING)countersis_node_meta_valid)(create_structured_trace_for_min_cut_info)is_with_effects)config)CustomKnapsackSolverCustomRuntimeEstimator)FakeScriptObject) is_builtin) LazyStringtrace_structured) trace_log)extract_tensor_metadata) BackwardState) is_sym_node py_sym_types) magic_methodsmethod_to_operator)find_symbol_binding_fx_nodes free_symbolsis_symbol_binding_fx_nodeoptimization_hintstatically_known_falsestatically_known_true) graph_drawer) OrderedSet)CheckpointPolicy)GraphInfoProvider) dp_knapsackdp_knapsack_sliding_hirschberggreedy_knapsack ilp_knapsack)KnapsackEvaluator) AOTOutputSavedForBackwardsAOTOutput#SavedForBackwardsNoVcCheckAOTOutput)_is_functional_graph)is_opaque_node)get_aot_graph_name)_is_bwd_seed_offset_is_fwd_seed_offset _is_primal _is_tangentget_cuda_generator_meta_val) fx_graph_cseget_aten_targetraise_getitemsboolAOT_PARTITIONER_DEBUGzlogging.LoggerlogcleZdZUdZded<ded<ded<ded<ded<ddZdd Zdd Zdd Zdd Z y )OpTypesz8Class for keeping track of different operator categorieszOrderedSet[Callable[..., Any]] fusible_opscompute_intensive_ops random_opsview_opsrecomputable_opsc0t||jvSN)r9r@selfnodes \/var/www/html/ia-enhanced/venv/lib/python3.12/site-packages/torch/_functorch/partitioners.py is_fusiblezOpTypes.is_fusiblehst$(8(888c0t||jvSrF)r9rArGs rJis_compute_intensivezOpTypes.is_compute_intensivekst$(B(BBBrLc0t||jvSrF)r9rBrGs rJ is_randomzOpTypes.is_randomnst$77rLc0t||jvSrF)r9rCrGs rJis_viewzOpTypes.is_viewqst$ 55rLc0t||jvSrF)r9rDrGs rJis_recomputablezOpTypes.is_recomputabletst$(=(===rLNrIfx.Nodereturnr;) __name__ __module__ __qualname____doc____annotations__rKrNrPrRrTrLrJr?r?^s=B//99..,,449C86>rLr?ceZdZUded<ded<ded<ded<ded<ded <ded <ej dd Zdd Zdd ZddZ ddZ y)NodeInfo list[fx.Node]inputsOrderedSet[fx.Node]_required_fw_nodesrequired_bw_nodestangents_closureunclaimed_nodesdict[fx.Node, int]fw_orderstatic_lifetime_input_nodescFtdjDfdS)Nc3 K|]}|ywrFr].0ns rJ z-NodeInfo.required_fw_nodes..s 01Q 0s c"j|SrF)rh)rnrHs rJz,NodeInfo.required_fw_nodes..s a@PrLkey)sortedrcrHs`rJrequired_fw_nodeszNodeInfo.required_fw_nodess! 0// 06P  rLc||jvSrF)rcrHrns rJis_required_fwzNodeInfo.is_required_fwsD++++rLc||jvSrF)rdrxs rJis_required_bwzNodeInfo.is_required_bwsD****rLc||jvSrF)rfrxs rJ is_unclaimedzNodeInfo.is_unclaimedsD((((rLcZ||jvrtd|d|j|S)NNode z not in fw nodes!)rcAssertionErrorrhrxs rJ get_fw_orderzNodeInfo.get_fw_orders4 D++ + 5+ >}}QrLN)rWr`rnrVrWr;)rnrVrWint) rXrYrZr\ functoolscached_propertyrvryr{r}rr]rLrJr_r_xsZ ++**))((  !44  ,+) rLr_c@eZdZUded<ded<ded<ded<ded<y) MinCutOptionsr;ban_if_used_far_apartban_if_long_fusible_chainsban_if_materialized_backwardban_if_not_in_allowlistban_if_reductionN)rXrYrZr\r]rLrJrrs  $$"&&!!rLrcz|jjddtjtjfvS)N recompute)metagetr%MUST_RECOMPUTEPREFER_RECOMPUTErIs rJmust_recomputers5 99==d +''))0 rLcT|jjD]}t|syy)NTF)graphnodesrfx_grIs rJhas_recomputable_opsrs+    $  rLc|jjD]W}t|st|jds&t j j|jjvsWyy)NtagsTF) rrrhasattrtargettorchTagnondeterministic_seededrrs rJhas_recomputable_rng_opsrsU    4  V, 11T[[5E5EE  rLc t|jdtjtjfryt|jdtj s$t dt|jdy)Nvalr&z.expected node.meta['val'] to be SymFloat, got ) isinstancerrSymIntSymBoolSymFloatrtypers rJ sym_node_sizersg$))E"U\\5==$AB dii& 7|jtjjjj tjjj j fvry|jd}t|tjsyt|S)zCheck if node is a view/reshape of a higher-order op output that aliases an input. Returns the original input node from the higher-order op's kwargs if the pattern matches, None otherwise. Nr) rropsatenviewdefaultreshaperrrrr)rIsources rJ_is_copy_node_bw_onlyrsi  {{599>>..66 8N8N8V8VWW YYq\F fbgg & $V ,,rLct|}|||vrt||ts||St|}|||vrt||ts||Sy)aTry to find a valid input replacement for an invalid forward output. This handles cases where a forward output depends on backward nodes but semantically aliases an input. For example, a view of a getitem from a triton kernel that mutates a buffer in backward, or a direct getitem from such a higher-order op. The original input may be a primal or a valid intermediate node already present in the forward graph. N)rrrr)rIenvrs rJ_find_input_for_invalid_outputrsk+40N" c !3~.@>"".t4N" c !3~.@>"" rLFcxtj}i|D]3}|j|j}|j|_||<5|j D]:}|s.Z CFrLget_attrc|SrFr]rs rJrqz4_extract_graph_with_inputs_outputs..]rrLoutputrz couldn't be found in envNr&rz was invalid, but is outputdesc stack_traceoutput_stack_traces)$rGraphrnamerr_must_be_in_backward InvalidNode_must_be_in_forwardrpytreearg_tree_leavesrrrrrany node_copyzip RuntimeErrorrrrrcopy_rlenrappendrrtuplereliminate_dead_codelint) joint_graphraoutputs outputs_descssubgraphignore_must_be_in_fw_bw new_graphrInew_nodeall_argsr output_valuesx_desc replacementoutvrs @rJ"_extract_graph_with_inputs_outputsrsm$ I"$C((3  D !!)&$T* *&'D $D) )&'D  3;  WW %#CI WW '--tyyHDKKHH"a)3q6?3H 8}'D !++D2BCCI WW "!++D2BCCI WW S)TM-0$ 6 a !|"U1#-F#GHH#a&/2# HH  4 4 < <<,Q/AFF q("166!9bgg6q S(&s166!9~G"%affQi.K&"@C"HK*!((5$uQC/J%KLL  Q (   #=$>   5/ 0C$CHHV ' &0277%; =!E'CHH "# !!# NN uf's 84N2>N7ctjxryt|jtj j xrt|j xs1|jtjjjk(SrF) ris_non_builtin_to_includerrr_ops OpOverloadrr higher_order triton_kernel_wrapper_functionalrs rJrrs`  + +  DKK!6!6 7 W 4;;@W.s K$)) Krrr) rrr find_nodesnextiterrrr) joint_modulenum_fwd_outputsrr fwd_outputs bwd_outputsfwd_outputs_descsbwd_outputs_descss rJ_extract_fwd_bwd_outputsr&s$$ K 2 2 = = = J KG** T,$$//8/< =>CCGG TFS\) M *?+K/*+K%&67%o&67  %68I IIrLcV|D]$}|j|k(s|j|yyrF)rremove) saved_valuesr saved_values rJ_remove_by_namer+s0#   t #    , rLct|}tt|dz ddD]}t||r|dz}|S|S)Nr&)rranger)fwd_module_outputsidxis rJfind_first_sym_noder2sX  !C 3)*Q.B 7-a01a%C  J  JrLc  |j|5|jtjjj j |f}tjjj j |jd|jd<t|jd|jd<ddd|j5|jtjjjj |dgdf}tjjjj |jddgd|jd<t|jd|jd<ddd|j5|jtjjjj |tjf}tjjjj |jdtj|jd<t|jd|jd<ddd|j5|jtjjjj ||f}tjjjj |jd||jd<t|jd|jd<ddd|j5|jtjjjj |f} tjjjj |jd| jd<t| jd| jd<ddd|j 5|jtjjjj | |f} tjjjj!| jd|| jd<t| jd| jd<ddd|j 5|jtjjjj | tj"fd|d|j$} tjjjj | jdtj"| jd<t| jd| jd<ddd| S#1swY+xYw#1swYfxYw#1swYxYw#1swYxYw#1swY xYw#1swYHxYw#1swY SxYw) Nrr tensor_metar-Tfp8_scale_pos__rr)inserting_afterrrrrabsrrramaxprimsconvert_element_typefloat64 clamp_min reciprocalmulTensorfloat32r) rrImaxminpositionabs_node amax_node amax_64_nodeclamp_min_nodereciprocal_nodemul_node scale_nodes rJcalculate_quantization_scalingrMs$   t $U&& IINN   & &'  %yy~~1199$))E:JK e'>x}}U?S'T m$ U   x (W'' IINN   ' 'RD$'( !&  3 3 ; ; MM% 2$! u)@ u@U(V }%W   y )  ** IIOO 0 0 8 8U]]++ $)99??#G#G#O#O NN5 !5==$ % ,C   e $, -(     | ,  ,, IINN $ $ , ,$- &+YY^^%=%=%E%E   e $c& E".E    &. M*     ~ .  -- IINN % % - - ". ',iinn&?&?&G&G    &' U#/F   '/ ]+      /U&& IINN   % %!3''  %yy~~1188   '  e(?x}}U?S'T m$U   x ( Y(( IIOO 0 0 8 8EMM*!(1TYYK8) "'!E!E!M!M MM% %--" *AQVAW)X  & Y IUUWW         UU Y sZB3W.B9W;0CXB5X(B3X"4B5X/C"X<.W8;XXX"X,/X9<Yc  |j|5|jtjjj j |tjf}tjjj j |jdtj|jd<t|jd|jd<ddd|j5|jtjjjj||f}tjjjj|jd|jd|jd<t|jd|jd<ddd|j5|jtjjjj ||f} tjjjj |jd|| jd<t| jd| jd<ddd|j 5|jtjjjj | |f} tjjjj | jd|| jd<t| jd| jd<ddd|j 5|jtjjj j | |fd|d|j} tjjj j | jd|| jd<t| jd| jd<ddd| S#1swYbxYw#1swYxYw#1swYxYw#1swYxYw#1swY SxYw)Nrrr4fp8_quant_pos_r6r7)r8rrrr;r<rrBrrrr@rAr> clamp_maxr) rrIrL quant_typer>rPrEtarget_node_32scaled_target_nodeclamp_min_scaled_nodeclamp_max_scaled_nodequant_activation_nodes rJperform_quantizationrW4s   z *  ,, IIOO 0 0 8 8 &- &+YY__%I%I%Q%Q IIe emm& E".E    &. M*     ~ .  "00 IINN   % % *-1 */););)B)B    & (>* &2I  # #E *2  .     1 2  % 3 3 IINN $ $ , ,$i0!4! -2IINN,D,D,L,L  # #E *I- ""5)5L ! & &u -5 ""=1     4 5  % 3 3 IINN $ $ , ,'3!4! -2IINN,D,D,L,L ! & &u -y- ""5)5L ! & &u -5 ""=1     4 5  % 3 3 IIOO 0 0 8 8'4!(1TYYK8!4!  IIOO 0 0 8 8%**51:  ""5) 5L ! & &u -5 ""=1   ! u               ! s@CQL&&(L < 'K 88rLc tjjjdj dd}|j dDcgc]$}t t|j dd&}}|Scc}w)N!activation_quantization_aten_passallowed_dtypesztorch.bfloat16;.r-)r _inductorrpost_grad_fusion_optionsrsplitr)r`dtypes rJget_allowed_dtypesrgsz__++DD+ c ,-;I:N:Ns:S16u{{3'+,N s )A8cBt}t|r|jdj|vrytj j jdjdd}t|jd}tj j jdjdds||k\Stj j jdjddrt||k\xst||k\ St||k\S)NrFr_ size_in_mbdskip_dynamo_guardsquantize_dynamic_shape) rgr rrfrrcrrdrr]r"r!)rIr`size_thresholdris rJshould_quantizerns')N d #tyy'7'='=^'S__++DD+ c,'tyy'78J ?? ! ! : :+  c & '^++ ?? ! ! : : / #& . /)n,J+J.,HII J )~)EF FrLctjjjdj dd}t t|j ddS)Nr_rQztorch.float8_e5m2rbr-)rrcrrdrrre)rQs rJget_quant_typerpsN''@@+ c,+, 5***3/3 44rLc\tj|}|j|jfS)z Calculate the range of values for a given torch.dtype. Args: dtype (torch.dtype): The input dtype. Returns: tuple: A tuple containing the minimum and maximum values. )rfinforDrC)rfinfos rJcalculate_rangerts% ;;u D 88TXX rLc |jdd}|jd}t}t|\}}t }g}g} t |D]\} } | |kr | j jdds*tjjjdjddrOt|| |d | } t|| | |||| } t| s|j| n| j| n|j!| 5|j#tj$j&j(j*| |fd | d | j, } tj$j&j(j+| j d || j d <t/| j d | j d<ddd || <t |D cgc]\}} |j|| }}} t1|}|| z}|r|d||z||dz}|j3dt5|t6ddxxdz cc<y#1swYxYwcc} }w)Nrrrsaved_for_quantizationFr_ use_scalingT-q=rOr6r7rr4inductor%activation_quantization_fwd_aten_passr&)rrrprtdict enumeraterrrrcrrdrMrWrrr8rrr;r<rrrr2 update_argrr )rr!rr"rQr>rPposition_to_quanttensor_scale_nodessym_scale_nodesrErIrL quant_noder1output_updated_argsr0 scale_nodess rJquantize_activation_fwrs     *1 -F++a.K!J*:6Iy(*%'O#K0-5$ o %  99==15 9%%>>3c-&'<4E8 24ZIx #:.&--j9#**:6**40 !&!4!4 <<DD"J/-hZq D"5"J  <<DD IIe,jOOE* 6M".6JOOM2 +5 h '[-5d7@ 6L+21da& 1 2C$6K  % 36I#$6O O  a234 Z@AQFAA  &sCI*;I6*I3 c  |jDcgc]}|jdk(s|}}d}|D]~}|jjdds!|jj d|jj d}t j jjdjddr|j|5d|jjdd z t fd |D}ddd|j5|jt jjj j"||f }t jjj j#|jd ||jd <t%|jd |jd <ddd|j|5|jt jj&j(j*||f }t jj&j(j+|jd |jd |jd <t%|jd |jd <ddd|j5|jt jjj j"||f }t jjj j#|jd ||jd <t%|jd |jd <dddn|j|5|jt jjj j"||fdt-|jz}t jjj j#|jd ||jd <t%|jd |jd <dddt/|j0j3D] }|k7s ||k7s|j5||"t6ddxxdz cc<ycc}w#1swYxYw#1swY xYw#1swY;xYw#1swYxYw#1swYxYw)NrrvF dequant_typer_rw fp8_scale_ fp8_quant_c3@K|]}|jk(r|ywrFr)rm bwd_input scale_names rJroz)quantize_activation_bw..s%&%$>>Z7"&srrr4dequant_r7ry%activation_quantization_bwd_aten_passr&)rrrrpoprrcrrdr8rrrrrr;r<rrrdivrArlistuserskeysreplace_input_withr ) rrI bw_inputsactivation_noderrLdivided_target_node_32 dequant_nodeuserrs @rJquantize_activation_bwr si"'++J$M1IJIJOH@ 99==15 9 IIMM2 399==8L%%>>3c-'(**40!- 0A0A,PR0S!SJ!%&)2&"J**:6 &+&9&9 <<DD"L1':'O  <<DD IIe,l$((/ ;R',,U3;O((7 **?; -2-@-@ **11-z:.A.*:?9K9K9R9R',,U3Z__U5K:*//600F0K0KE0RS+// > **+AB #(#6#6 <<DD4lC$7$L  <<DD277> !%%e, 8O$))%08L%%m4  **40 #(#6#6 <<DD"L1'#dii.8$7$L  <<DD IIe,l!%%e, 8O$))%08L%%m4 TZZ__./ @<'DO,C++D,? @MH@T Z@AQFAYK        sJSS4S B5S*CS%B5S2C S> S S" %S/ 2S; >T ctddfdtj|tddfdtddfdjjd d jd }|D]}d |j vs|t jd d|j }jj|5jj|j }ddd|jd}jj|jd|jd<||jd<|j|jj|tjj j"dj%ddrt'jjd } | d} t)| D]} t+| r| } njjd d jd } | D]}d|j vsjj| 5jj|j } ddd jj|j| } t-jtddfdy#1swYxYw#1swYgxYw)Nartifactc dddS)N,before_activation_quantization_fwd_aten_passstringrencodingr]r]rLrJrqz5perform_fp8_activation_quantization..dB  rLc,jdddSNFT print_outputinclude_strideinclude_deviceprint_readable fwd_modulesrJrqz5perform_fp8_activation_quantization..h:44tD5 rL metadata_fn payload_fnc dddS)N+after_activation_quantization_fwd_aten_passrrr]r]rLrJrqz5perform_fp8_activation_quantization..qA  rLc,jdddSrrrsrJrqz5perform_fp8_activation_quantization..urrLc dddS)N,before_activation_quantization_bwd_aten_passrrr]r]rLrJrqz5perform_fp8_activation_quantization..|rrLc,jdddSrr bwd_modulesrJrqz5perform_fp8_activation_quantization..rrLrrrrz^fp8_quant_pos_\d+_rrrTrvr_rwrr-rc dddS)N+after_activation_quantization_bwd_aten_passrrr]r]rLrJrqz5perform_fp8_activation_quantization..rrLc,jdddSrrrsrJrqz5perform_fp8_activation_quantization..rrL)rrrrrrresubr8rrupdatereplace_all_uses_with erase_noderrcrrdrrreversedr6r)rrbwd_module_inputsr!quant_fwd_module_outputsfwd_noderquant_bwd_inputrquant_bwd_module_inputs bwd_input_locbw_inputscaled_fwd_module_outputsscale_bwd_inputs`` rJ#perform_fp8_activation_quantizationr\s   :++_=     *//::h:GJOOPQR, 3 8== ()-r8==AI!!11)< S","2"2">">HMM">"R S$>>.9L  ' ' 6=AO !9 :3?O  0  + +O <    ' ' 2 3 66+ c-#'z'7'7'B'Bm'B'T"U/3  !89 Hx( (   %/$4$4$?$?8$?$LQ$O$T$TUV$W!1 0Hx}},%%55mDW&0&6&6&B&B &B&VOW$$++HMM: /  0:++,  ? S S0WWs'K'K%K" %K. c|r|Dcgc]}|jc}ng}|Dcic]}|j|}}tjjjdj ddr)|Dcic]}d|jvs|j| }}|j jddjd}|j jdDcic]}|j|} }d} |D]}|j|vst|s|j|vr!tjd |jMd |jd <|jd j|jd <d | |jjd <|jd j| |jjd <d } | rt||| |yycc}wcc}wcc}wcc}w)Nr_exclude_primalsFprimalsrrrrz*Skipping quantization of static input %s: Trvrr)rrrcrrdrrrrrnr=debugrrfr) r)rrrir!rIstatic_input_namessaved_values_namesr/rshould_perform_fp8_quants rJenable_activation_quantizationrs ' ;;t;  7CCd$))T/CC 66+ c U#$)5 $ 8RDIItO  $))444A!DII!L$.$4$4$?$?=$?$Q  4 %" , 99* *t/Dyy.. F R26DII. /(, %(8(>(>DIIn %JN dii ( - -.F G@D %@P@V@V dii ( - -n ='+ $ , +  $5  9 <D sG$G)4G.G.#G3)riromit_aot_autograd_runtimec B t||\}} } } |jjd} gtt| } |rgngtt | }gtt | }gtt| }gtt| }|g}t|j||z|z|z|z| | d|}tjj}|jdD]}|jsCt||jt||jt||jR|r_t!d|jDrCt||jt||jt||jt|st||j|rt#dt%}g}g}|D]C}t'|}|r#|j)||j+|3|j+|Et-|j}t/j0|||D]]}d |j2vrt5|j2d |z }t7|d D]}||vr|j+||||z}_|j9|j;||z|sg}g}g}|D]|}t=|j2j?d t@r|j+|>|j2j?d d r|j+|l|j+|~|j9|j;||ztC|}tE|D]D\} }| |k\s |j2j?d d r)t#d| d|dtC|t|j| |z||z|z|z|z| tGtC|tC|ztC|ztC|zD cgc]+} | |k\r| tC|kr tI| n tK| -c} zd|}!t|j||z|z|z|z|z|z| | d|}nt|j| |z||z|z| tGtC|tC|zD cgc] } tK| c} zd|}!t|j||z|z|z| | d|}tLjNjQ||!}"tLjNjQ||}#tRjTj?dd tW||"|#|||"|#fScc} wcc} w)ajExtract forward and backward graph modules from a joint graph. Args: ignore_must_be_in_fw_bw: When True, disables forward/backward placement enforcement in _extract_graph_with_inputs_outputs. Needed when the joint_module is not an original fwd+bwd joint graph (e.g. a backward graph being re-partitioned for dI/dW splitting). omit_aot_autograd_runtime: When True, skips postprocessing that is only needed when the resulting modules will be wrapped in a custom autograd.Function (the AOTAutograd path). This includes: tangent input handling, version-counter check sorting of saved tensors, opaque object (FakeScriptObject) separation, and fp8 activation quantization. Set this to True when the fwd/bwd modules will be executed directly without autograd. r!rrNr)rc3K|]X}|jtjjjj uxrt |jdk(ZywrN)rrrr wait_tensorrrrrls rJroz+_extract_fwd_bwd_modules.."sS)  HH 22>>FF F "AGG ! ") sAA z'backward_state_inputs must not be emptyrc|jSrFr)ss rJrqz*_extract_fwd_bwd_modules..Js 166rLrrsaved_tensor_with_no_vc_checkFzi=z, no_vc_check_start_idx=z, len(saved_values)=rr_),r&rrfilterr5r6r4r3rrr distributed is_availablerr+rallrr$raddrr itertoolschainrrrtclearextendrrrrr|r.r/r.r_lazy_graph_module_make_graph_moduleinductor_configrdr)$r r)saved_sym_nodessaved_opaque_nodesr!rirrr"r#r$r% placeholders primal_inputstangent_inputsfwd_seed_offset_inputsbwd_seed_offset_inputsbackward_state_inputs bwd_graphdistributed_enabledrI saved_symbolssaved_sym_nodes_bindingsaved_sym_nodes_derivedsymbolsymbol_bindings new_symbolsrsaved_values_with_vc_checksaved_values_no_vc_checksaved_opaque_objectsno_vc_check_start_idxr1 fwd_graphrrs$ rJ_extract_fwd_bwd_modulesrsW4 !OCK/1B %%00M0BL7fZ67M'-Qvk  ? $  %"25LLM $&("#% !  8D$))--.0@A$++D1>F(//5*11$7 8 69QQR #$> ?!. GAt))yy}}%DeL(QC78M7NNbcfgsctbuv 7    2 2  " #! !     %./0,-./*+ --!c,6G2G4A6/23   $;+ .7     " #! !   %  % $  $   $;  7    2 2 , & 8 s<033GGH+1-  $;  7     $ %$ $   $;  &&99, RJ&&99, RJ0044 /   '    '    z !!K Fs 90V!V)static_lifetime_input_indicesric g}d}|jjD]&}t|st|s t |s%|}(| t d|jjD]$}t |s|j|||us$ntd|D}t|} t|} | rIt|jd$tjdt||||St|d}t j"s t%|t'|t)||g}t+|||} g} g} g}t,j.j1dd }dd }dfd }|jjD]}|j2|vr|j4d k(r$|j2d |j7DvrF|j8t,j:j<j>j@t,j:jBjDj@t,j:jBjFj@t,j:jBjHj@t,j:jBjHjJfvr7tM|r| j|U||r_|jNjQdtRjTk(r0tW|r|j|n| j|||rM| rt d|d|j8tW|r|j|n| j|tW|r|j|.||s|j4dk(rt d|d|jXDcgc]}|j2|vs|}}t[d|Dr| j]|t_|r| j|tatbje| jg} tatbje| jg} tatbje|jg}t jhrti|j| } | | jj}tm|| | |||\}}|jjotp|jjotp| r'| rts|||tu| \}}tw|}t jxrddl=m<}|||||t}|}t}|}t|d}tu| jdkDr t|d}||fScc}w)a Partitions the :attr:`joint_module` in a manner that closely resembles the behavior observed in the original ``.forward()`` and ``.backward()`` of the callable, i.e., the resulting forward graph contains those operators that are executed in the original ``.forward()`` callable passed to :func:`aot_function`. The default partitioner collects the operators that are between the forward inputs and the forward outputs. This helps in finding the tensors which have to be stashed for the backward pass. These stashed tensors become the output of the generated forward graph. The remaining operators are then placed in the backward graph. .. warning:: This API is experimental and likely to change. Args: joint_module(fx.GraphModule): The joint forward and backward graph. This is the result of AOT Autograd tracing. Returns: Returns the generated forward and backward Fx graph modules. Nzlast_node must not be Nonec3TK|] }|jdk7s|j"ywrNrrrs rJroz$default_partition..s$$tww(/B $((rzxTrying to unsafely apply AC to a non-functional graph with the default partitioner. Falling back to min-cut partitioner.)r!rTis_default_partitioncd|jvxs=t|jjdtjj S)Nr4r)rrrr _subclasses FakeTensorrs rJ is_tensorz$default_partition..is_tensor#s= ) Z IIMM% %"3"3">">.  rLcntd|jDxrt|jdkDS)Nc3VK|]!}|jtjk(#ywrF)rrrrmrs rJroz=default_partition..is_multi_output..*sGD x///Gs')r)rrrrs rJis_multi_outputz*default_partition..is_multi_output(s. GDJJG G $DJJ!# rLc|jdxrO|jdvxr? xs:|jtjj j juS)NF) impure_random)rr) is_impurerrrrrrr)rIrs rJrz$default_partition..is_impure.sc NNN /  ('U;;eii&@&@&L&L&T&TT rLrc3&K|] \}}| ywrFr])rmkrs rJroz$default_partition..Cs3 !QA3 srz.Trying to apply AC on a graph with impure op: z, rz Expected z to be a tensorc32K|]}t|ywrFrrls rJroz$default_partition..ps7!{1~7rrr!ri)is_impure_noder&enable_activation_offloadingFrrU)Arrr r5r4rr6rr$rrr0warningswarn#min_cut_rematerialization_partitioncleanup_recompute_tagsr(unsafe_allow_optimization_of_collectivesforce_save_collectivesforce_save_effectful_opsforce_save_bw_mutation_srcclassify_nodesrrrrr named_modulesrrr_assert_scalarrprofiler_record_function_enter_new_record_function_enter_record_function_exit_RecordFunctionrrrr% MUST_SAVEr1rrrrrr{fromkeysr_sync_decision_cross_ranksrirrrfunctionalize_rng_opsr#reordering_to_mimic_autograd_enginer,_activation_offloading.activation_offloadingr:thread_graphsafe_rng_from_hopsrd)r  _joint_inputsr!rri forward_nodes last_noderIforward_node_namesgraph_has_recomputable_opsgraph_has_recomputable_rng_ops node_infor)rrr rrrnbackward_usages fw_module bw_modulerrs @rJdefault_partitionr?s\@MI""(( t $ 4(8>@AOdmm,>?DDFG ((1,2D2DlS "*&/&K&K#3'-'$? IyOO''7H'I OO''7H'I! )#8iC4H$ Iy8 B **  %    '  y)Iy)I.yeLI 9 & &'!+29$O i EVs .W0W0g.Ac ||jzSrF)itemsize)rYrfs rJ_tensor_nbytesrBs 5>> !!rLcJd dd|jvr|jd}t|tryt|ttfrt fd|DSt|t r"t fd|jDSt|tjr|Stdt|d||jdk(s:|jtjjj j"ury td |d ) Nct|tjsytt |j d|j S)Nrfallback)rrrArBr rYrfrs rJ object_nbytesz_size_of..object_nbytess2!U\\*/ DI177SSrLrr&c3.K|] }|ywrFr])rmrnrIs rJroz_size_of..s5A}Q'5c34K|]\}}|ywrFr])rmr6rnrIs rJroz_size_of..s@DAq}Q'@zUnknown metadata type z on node rrrzO didn't have `val` metadata; we should always have `val` metadata on the nodes.)robjectrWr)rrrrrsumr{itemsrrArrrrrrr(r)rIrrIs @rJ_size_ofrQsT   ii c< (dE] +555 5 T "@CIIK@@ @ U\\ * % %3DI;ivNOO ww* uyy~~/L/L/T/T T  vde rLc 2ddlm}|t}|jD]3}|jdk(s||j j xxdz cc<5tjdt|jtjddy)Nr)rrr&%sTrsreverse) collectionsrrrrrrXr=rsrtrPr itemgetter)rrcntrIs rJ _count_opsrYsr'%c*C + 77o %  $$ % * %+HHT6#))+8+>+>q+A4PQrLcg}ttjjD]}t tjj|}t |tj jsL|jD]G}t ||}tjj|jvs6|j||SrF) dirrrrrrrOpOverloadPacket overloadsr pointwiserr)r attr_nameopoverloadpacketoverload op_overloads rJ pointwise_opsrcs-/C(  "599>>9=*EJJ,G,GH (224 H!"2H=Kyy""k&6&66 +,    JrLc|Dcic]7}t|tjjjs2|||9}}t |j tjddScc}w)Nr&TrT) rrrrIrrtrPrrW)r depth_maparg arg_depthss rJ sort_depthsrhsh(, #z#uxx}}?Q?Q/RYs^J *""$(*=*=a*@$ OOs 3A2A2c tj i |jjdD]} j | fd |<t |jj Dcic]\}}|| c}} d fd }ttt|jj }d}tj}|D]#}|jD]} ||ks |}|}%||St|jj d |D]U}|jdk(s|jtj j"j$j&usN||Wt|jj  |dD] }|| tjj)| } | Scc}}w)a This pass finds the first bwd node in the graph (by looking at users of tangents) and then reorders the graph by walking from this node to all the way to the end of the graph. At each op in this traversal, we insert this op in a new graph and try to bring only the relevant subgraph from the other non-bwd edges relevant for this op. This closely mimics the behavior of autograd engine. Why is this pass required in the first place? This is an artifact of how partitioners work today. The starting point of partitioner is a joint graph, which is fwd and then bwd graph. In the case of checkpointing, we keep portions of fwd graph in their original place in the joint graph, while obtaining a bwd graph. As a result, the resulting bwd graph has copies of recomputed fwd subgraphs followed by the original bwd graph. If we run this naively, this leads to bad memory footprint, because the fwd subgraphs are live for way longer duration than necessary. This pass reorders the operations such that we prioritize the ops for the original bwd graph while only realizing those ops from the fwd graph that are necessary at any given point in the graph. rrc|SrFr]rs rJrqz5reordering_to_mimic_autograd_engine..s ArLc*|g}t}t|dkDrH|j}||vs|vr'|j|||jz }t|dkDrHt |fd}|D]}j |fd|<y)Nrc|SrFr])rnorders rJrqzSreordering_to_mimic_autograd_engine..insert_node_in_graph..+s %(rLrrc|SrFr]rs rJrqzSreordering_to_mimic_autograd_engine..insert_node_in_graph..-rrL)r$rrrall_input_nodesrtr)rI cur_nodesinsertable_nodesrrrms rJinsert_node_in_graphzAreordering_to_mimic_autograd_engine..insert_node_in_graphsF 0: )nq ==?D''43;   & -- -I)nq ""28JK$ DD!++D2BCCI DrLNrrIrVrWNone)rrrrrr|rrrr6mathinfrrrrrrrr GraphModule) gmrIr0rrrfirst_node_in_bwd minimum_ordertangentrnew_gmrrrms @@@rJr2r2s. I"$C##}#5@''.>?D @)2"((..(A B93T3Y BED&&bhhnn=>NHHM!)MM )DT{]* %d $(! ))  RXX^^$%?u->'?@' 77o %$++9M9M9U9U*U  &'RXX^^$U+<%=%?@#T"#XX ! !"i 0F M[ Cs5 G c|j}| td|j} |j} tjj j } |jj|5|jjd|} t|| jd<| }ddd|jj|5|jjd|} t|| jd<| }dddt|j} |d<|jj|5| jd| |jg|j|}ddd|j!| j#|t|j} |d<| j%|5| jd| |jg|j|}|j!|| j#|ddd||fS#1swYzxYw#1swY+xYw#1swYxYw#1swY||fSxYw) a% Note [CUDA Graph Safe RNG Functionalization] CUDA Graph capture doesn't work with get_rng_state and set_rng_state because these functions operate on CPU values, while CUDA Graph RNG capture uses on-device CUDA tensors. To solve this, we use graphsafe_set_state with a CUDA Generator registered to the CUDA Graph before capture begins. graphsafe_set_state updates the generator's pointer to reference a different GeneratorImpl, ensuring subsequent calls are correctly forwarded to the desired generator (and its cuda-tensor RNG state during graph capture). For each RNG operation's forward/backward pair: - We create two generators initialized with identical values - Each forward and backward call advances its respective generator equally - This keeps generators synchronized so forward and backward operations use matching RNG values When forward is called multiple times before backward (causing desynchronization): - We save the forward RNG state - We update the backward Generator's state before executing backward Before each CUDA Graph replay, replay_prologue updates captured RNG pointers with current states, ensuring backward Generator changes are reflected during replay. This function modifies both forward and backward computation graphs by: Creating RNG state placeholders for both passes Updating the forward node to use graph-safe RNG state Updating the backward node to use graph-safe RNG state For more details: https://github.com/pytorch/pytorch/issues/113541 Nzdevice_idx must not be Nonefwd_rng_state_rbwd_rng_state_ rng_staterrr)indexrrr_prims rng_primsgraphsafe_run_with_rng_stater8rr7rr{r create_noderrrrinserting_before)r=r>fw_nodebw_nodedevice rng_countlast_fwd_inputlast_bwd_input device_idxfw_graphbw_graphr fwd_rng_state bwd_rng_state fw_kwargsfunctional_fw_node bwd_kwargs rng_outputs rJ%apply_graphsafe_rng_functionalizationrKs8RJ:;;HH#(<<#9#9#V#V   ( ( 8'!33nYK4PQ $? $K 5!&'  ( ( 8'!33nYK4PQ $? $K 5!& 'W^^$I*Ik  ( ( 1 %11  (..07<<0 2   !!"45  gnn%J+J{  " "7 +%))  (..07<<0 * %%j1G$% > ))M'' ''  % > ))s119H" 9H/.H<AI"H,/H9<IIc f 'tj}dd}dd'dd}||}||}||} i} |jjD]} t | st | j ds&tjj| j jvsW| j|vs| j| vrt|| j} || j} | | j}| |d| | <tjjj}tjjj}d}|jj!dD]} d| jvs| }n| t#d g}t%t'|jj!d}t%t'|jj!d}t)'fd | j+D}|j-tj.d t1|d kD}tj2j4}t4j6xr*| xr%|j8 xs|j:j<}t?| j+D]"\}}|d } |d}'| }|j}|j}|r'|%|j@dk(rtC||| |||||\}}Z|jE| 5|jGd|| j g| jH| jJ}|jGdtLjN|dfi}|||jPd<|jGdtLjN|d fi} tSjR| jP| _(| jU| |jW| |jY|ddd|jE|5dt%|}!|j[|!}"|||"jPd<ddd|jE|5|jGd|"|j g|jH|jJ} |jU| |jW|ddd%|rt%t]|jj!d}#|#jHd}$t1|$|z }%|$d|%t_|z|$|%dz}&|jja|&|jjW|#|jc|jc||fS#1swYxYw#1swYLxYw#1swYxYw)Nci}|jjD]i}|jdk(st|jds*t j j|jjvs[|||j<k|S)Nrr) rrrrrrrrrr)gmod random_nodesrIs rJ get_rng_opsz*functionalize_rng_ops..get_rng_opssl+- JJ$$ /D?*DKK0II559I9II*. TYY'  /rLc d|jvry|jd}t|ts|f}|D]D}t|tjs|j j dk(s8|j cStj dS)zV Check the example value of the node outputs to find the device type. rNcudacpu)rrrrrArr)rI candidates candidates rJ get_devicez)functionalize_rng_ops..get_devices  !YYu% *e,$J# ,I)U\\2##((F2$+++ , ||E""rLc:ddlm}|}| td|5|E|jdk(r6|j t j jcdddS|j t jcdddS#1swYyxYw)Nr)detect_fake_modezfake_mode must not be Noner) torch._guardsrrr from_tensorrr get_rng_state)rr fake_modes rJget_sample_rng_statez3functionalize_rng_ops..get_sample_rng_states2$&   !=> >  @!fkkV&; ,,UZZ-E-E-GH @ @(()<)<)>? @ @ @s>B%"BBr)fwdbwdrrr{zaCouldn't find tangent node in graph inputs. This is unexpected, please file a bug if you see thisc34K|]}|dyw)rNr])rm node_pairrs rJroz(functionalize_rng_ops..s )2 9U#$rMrr&rrrrrrrrng_state_output_r)rfx.GraphModulerWdict[str, fx.Node])rIrVrWtorch.device | None)rrrW torch.Tensor)2rcountrrrrrrrrrrrrrun_and_save_rng_staterun_with_rng_staterrrrr$valuesdiscardrrrcrgraphsafe_rng_functionalizationfallback_random test_configs*graphsafe_rng_func_ignores_fallback_randomr|rrrrrrrrrcopyrrrrrrr recompile)(r r=r> num_sym_nodesuidrrjoint_graph_rng_opsfw_graph_rng_opsbw_graph_rng_opsrecomputable_rng_ops_maprI base_noderrrun_and_save_rngrbw_tangent_start_nodefw_rng_state_outputsrrdevicesmulti_cuda_devices ind_config'use_rng_graphsafe_rng_functionalizationrrrrrrstater state_namebw_rng_state_nodefw_output_node fw_outputssym_node_start_idxrrs( @rJr1r1s2 // C #$ @&l3"9-"9-!""((S 4  V, 11T[[5E5EE yy 00DIIEU4U+DII6I&tyy1G&tyy1G:A'2R $Y /S ||--DD//BB **m*<  !$( ! $ o  +-(9??#=#=#=#OPQN(9??#=#=#=#OPQN6N6U6U6WG OOELL'(W)''J.. " " ** * R&&QQ ,!**B*I*I*K LP- 9E"E"G$???? 4" v%-R . *NN**73! 3%-%9%9#$  #>>&: &"!,,#$$,a0 - %9$@ 5!%11#$$*2 #'))GLL"9 --j9##G,$++E2C! 3H**+@A M0c < $,$8$8$D!0DV0L!&&u- M **73 -%11#&*! #>>2  --j9##G, - -EP-jd9??#=#=#=#JKL#((+  _}< ** +() *+,- .  w'"">2   i S! 3! 3H M M  - -s&C.V 5V(AV& V V# &V0 c|jjD]t}t|jtj j s2|jjdk(sLt|rXtj|jd<vy)z By default, the partitioner is not allowed to recompute collectives unless they come from a user-annotated AC region. See Note [Recomputing collectives in the partitioner] rrN) rrrrrrrrrr%r.r)r rIs rJr#r#sh ""((@ t{{EJJ$9$9 : %%);;"4(%5%?%?DIIk " @rLcdfd |jjD].}t|st|rt |r'|0y)a\ Force save outputs from with_effects nodes wrapping effectful ops. Effectful ops (registered via _register_effectful_op) should not be recomputed because they may have arbitrary global side effects (I/O, RNG state, collectives, etc.). We mark the tensor outputs of with_effects as MUST_SAVE to prevent recomputation of the effectful op. The with_effects node returns a tuple (token, result). We recursively find all leaf outputs extracted via getitem and mark them as MUST_SAVE. Since these are saved, the with_effects op doesn't need to be recomputed in backward. c |jD]t}|jtjus |t |j j dttfrXtj|j d<vy)Nrr) rrrrrrrrrr%r.)rIrmark_getitem_outputss rJrz6force_save_effectful_ops..mark_getitem_outputssaJJ HD{{h...$T*!$))--"6 F-=-G-GDIIk*  HrLNrs)rrrrr)r rIrs @rJr$r$sFH""((' D !"4((.  & 'rLct}t|jjD]}|jdk(r|j t jjjju}|rrt|r|j|jdt|s|jd|vstj |jdj"d<yy)Nrrr&r)r$rrrrrrrrrrrrrr r%r.r)r has_mutation_in_bwrIis_copy_s rJr%r%s5?L++112 77h  ;;%)).."6"6">">> +D1"&&tyy|4*40TYYq\EW5W1A1K1K ! !!+. !rLc|jtjk7ry|jd}t |t j urtdt |d|jvxr|jdk(S)NFrz#expected parent to be fx.Node, got r4r) rrrrrrrrrr)rIparents rJis_getitem_of_multi_outputrsh {{h&&& YYq\F F|277"B4<.QRR  + J?0JJrLc|jjD]T}t|r|jD]i}t|sd|jvsd|jvs-|jd|jdkDsMt j |jd<k|jjddstd|jDrt j |jd<d|jvstd|jDs t|rd|jdjvr4|s8t j |jd<W|S)a If there are two consecutive checkpointed blocks with no operator in between, we would still want to stash the tensor at the boundary of checkpointed blocks. The following pass makes the last output node non-recomputable to allow for that. ac_graph_idrhas_backward_hookFc32K|]}t|ywrFrrs rJroz)cleanup_recompute_tags..sE)-t$Erc32K|]}t|ywrFrrs rJroz)cleanup_recompute_tags..s@TN4(@rr) rrrrrr%r.rrrr)r rrIrs rJr!r!s3""((+@ $   H"4(%2%2 -0499]3KK-=-G-GDIIk* Hyy}}0%8E15EB&*:)C)C +&  *@TZZ@@+40]diiPQlFWFW5W$ &6%?%?DIIk "W+@X rLc =>?@ABCDEFGHIJKL ttJtrQtd|jD}|tdJjDz }t j d|dNd>dNd?dN>?Jfd B ddlHdOBJfd DdPDJfd }dOBfd C dQCJfd }HjIt=dRdS=IJfd } |jD]} | jdk(r| jvrp| jvr1Ij| jdzdt j"dn1Ij| jdzdt j"dt%| r1Ij| jdzdt j"dt'| r | | dnt)| r | | d|| } j+| r | r | | | d| j,vxrd| j,vxs8d| j,vxr(t/| j,dt0j2 } t5| rt7t9| } d}nj| rRt/| j,j;dt<t>fs tA| rd} d}n)t j"} d}n|| jB\} }|rO| t j"k(s | tDk(r3Ij| jdz| jdz| d|n.Ij| jdz| jdz| | jFD]?}Ij| jdz|jdzt j"dAdTBfd }jHr(jJD]}|jFDcgc]$}j+|rjM|&}}|jFDcgc]}j+|s|}}tO|dkDsw||tQ|}tS|jFD]x}j+|sjM||kDs*B||s4|=vr9t j d!|jM|||jM|| |zjTr^t}|jD]D}j+|sjM||fg}jM|}tO|dkDsJtWjX|\}}||vr,|j[|jM||d"zkDrNtO|dk(r@t j d#||jM|jM|| ||jFD]J}j+|sB||s|=vs$tWj\|jM||fLtO|dkDrG Hj^Id$d\}}|\}5Gt}6IfdE|5DD] \L}7|6jGLfdF|7D"t}8|6D]=\}9}:|9ddG|:ddHk7rtdI|9ddGdJ|:ddH|9ddG}-|8j[|-?t|Et|jD; cic]\};} | |; c} };FtEfdK|8DFfdLM}<|<=fS#t$r}td|d}~wwxYwcc}wcc}w#Hj`$r}tctdjf}d}dA|jh} | r| jkd%d&d&'n tm|Atod(d)Afd*+tqd,d-}ts|d.5}!|!juAdddn #1swYnxYwn#tv$r}d/|d0}Yd}~nd}~wwxYwd1jyHjzj|jI@tod(d2@fd3+tI}"|"ri}#d$g}$dUd4}%|"D]\}&}'}(|$j|'|&d$k(r*|%|'})|#j|)gj|(G|'dk(r*|%|&})|#j|)gj|(v|%|&|%|'k(r*|%|&})|#j|)gj|(|%|&}*|%|'}+|#j|+gjd5|*g},|#jD]5\}-}.|,jd6|-d7|.D]}/|,jd8|/7d1jy|,}0d9jy|$}1tI\}2KKrtod(d:Kfd;+|rd<|d1nd }3|2r |3d=|2d1z }3d }4|rd>}4td?|0d@|1d1|3|4|t j dAt j dBtHIfdCtId}~wtv$rEt j dAt j dBtHIfdDtIwxYwcc} };w)VNc3K|]H}|jdk(r7t|jdr!t|jjJyw)r_overloadpacketN)rrrrrrs rJroz solve_min_cut..sA& ww/)gdkkCT.U  ++ ,& sAAc32K|]}t|ywrF)rrmr1s rJroz solve_min_cut.."s4 CF4 rz&Ops banned from re-materialization: %scD|jtjjjk7ry|j d}tj jj|\}}|D].}|j|}||uryt|ts)||vs.yyNFrT) rrrrauto_functionalizedr_higher_order_opsauto_functionalizeget_mutable_argsrrr)ab mutable_opmutable_arg_namesr6rrfs rJ!can_fuse_into_auto_functionalizedz8solve_min_cut..can_fuse_into_auto_functionalized's 88uyy--AA AVVAY   # # 6 6 G G    & D((4.CCx#t$8  rLc|jtjjjk7ry|j d}|D])}|j d}| t d||}||us)yy)NFtensors_to_clonerzkwargs must not be NoneT)rrrrrrr)rrrrrrfs rJ.can_fuse_into_triton_kernel_wrapper_functionalzEsolve_min_cut..can_fuse_into_triton_kernel_wrapper_functional;sw 88uyy--NN NHH%78% D((8,F~$%>??,CCx  rLcbt|tjk(ry||ry||ry|jtj ur>|j djtjjjuryj|xrj|S)NTrF) r9rcatrrrrrrrrrK)rrrrop_typess rJrKz!solve_min_cut..is_fusibleHs 1  ) ,Q 2 9!Q ? HH(( (q   yy%%FFG ""1%@(*=*=a*@@rLrzANeed networkx installed to perform smart recomputation heuristicsc<j|ryt|g}t|dkDro|j}|jD]A}j |s ||syj|s1|j |Ct|dkDroyr)rRr$rrrryr)rIrpcurrrKr;rs rJis_materialized_backwardsz0solve_min_cut..is_materialized_backwardsbs   D !v& )nq --/C  ( //5jd>S##D)MM$'  ()nq rLcn|jdk7ry|jtjury|jj ddt jk(rytjrj|ry|jtjjtjjfvryjrj!|s1yj#|ryj%|ryt'|ryj(r3|r+t*j-d |t/|j0y |j2d kr|j2tj4kDry j6r0t9d |j:D}t=|}|dz|kryy)zRReturns reason string if node should be banned from recomputation, None otherwise.rNrzmarked MUST_SAVEznot in recomputable allowlistz random opzcompute intensive opznon-builtin opzmaterialized backwards: %s %szmaterialized in backwardiztoo far from backwardc3hK|]*}t|tjst|,ywrF)rrrrQrs rJrozBsolve_min_cut..should_ban_recomputation..s&% !*Q2H %s22rz reduction op)rrrrrrr%r.rrecompute_viewsrRrlift_fresh_copyr lift_freshrrTrPrNrrr=rrr dist_from_bwmax_dist_from_bwrrOrrQ)rIinput_tensors_size output_sizermin_cut_optionsrs rJshould_ban_recomputationz/solve_min_cut..should_ban_recomputationpsr 77o % ;;(** * 99==d +/?/I/I I%  ! !h&6&6t&< ;;4//779P9PQ Q  2 2++D16!!$'",,T2-(.'  7 7.is_materialized..sE$z$-Es)rrr)rIrKs`rJis_materializedz&solve_min_cut..is_materializeds* 77m #E$**EEEErLc tjr||vryt|}tjr#j |rt j dfSt|jdtr/t|jdtjstdfSt|dtt|j ddzz}|r|dfS|d zdfS) zReturns (weight, cannot_save_reason). cannot_save_reason is None for finite weights, or a string explaining why the node cannot be saved for infinite weights. rzview op (recompute_views=True)rz$SymFloat (non-SymInt symbolic value)g?rjr&N)r treat_parameters_as_free_to_saverQrrRrurvrrrrrINT_INFrrCrDr)rIrimem_szrrs rJget_node_weightz&solve_min_cut..get_node_weights  3 333$  ! !h&6&6t&<88== = dii& 5dii. = FFF cST%6%6!.ban_recomputation_if_alloweds   D ! 8 4;; (=(=>@KK))-?? >>m $  DII *TYYu-=u~~"N   II XX4:'x0  AS  rLr_outsinkz;must be available for backward: input required for gradientrrz3must be computed in backward: required for gradientz+must recompute: marked by checkpoint policyz primal inputzforward RNG seedrr4znon-tensor outputz cannot save: )rzdata dependencycg}|D]*}tj| j||df,t|dkDrtj|\}}}|s j|S|j D]_} j |s j||kDr* j|| ||f}||vsJtj||at|dkDr|S)z Finds the first unfusible node in the chain of nodes starting from `start_nodes` and returns its position. Tr)heapqheappushrrheappoprry) start_nodes max_range sorted_nodesrnr6rInode_is_fusiblerrrKr;s rJfind_first_unfusiblez+solve_min_cut..find_first_unfusibleh s 9;  OA NN<)*@*@*CQ)M N O,!#',}}\'B $At_" --d33 :++D1 --d3i? !..t4"4.6C ,.|S9 : ,!#rLz1used above/below fusible %s:(%s) -> %s -> %s:(%s)rjztoo long %s %s %s %srFTrrc dddS)Nmin_cut_failed_fx_graphrrr]r]rLrJrqzsolve_min_cut.. s5 (%rLcSrFr]) fx_graph_strsrJrqzsolve_min_cut.. s<rLrmin_cut_failed_graphz.txtwz(failed to write: ) c dddS)Nmin_cut_failed_edge_listrrr]r]rLrJrqzsolve_min_cut.. s2$!rLcSrFr]) edge_list_strsrJrqzsolve_min_cut.. s}rLcXdD]$}|j|s|dt| cS|S)N)rr)endswithr) node_namesuffixs rJ get_base_namez$solve_min_cut..get_base_name s:-9F ))&1(CK<889! rLz depends on z :z - z -> c dddS)Nmin_cut_failed_svgrrr]r]rLrJrqzsolve_min_cut..9 s 4$,)rLcSrFr]) svg_contentsrJrqzsolve_min_cut..= s{rLzFX graph dump: zMin-cut graph visualization: z[Production debugging: Use tlparse to extract debug artifacts (min_cut_failed_fx_graph, min_cut_failed_edge_list, min_cut_failed_svg)] aAOT Autograd failed to partition the joint forward-backward graph. The partitioner determines which intermediate values to save from the forward pass vs recompute in the backward pass. This error means a value is required for backward, but cannot be saved AND cannot be recomputed. This is a bug in PyTorch. Please file an issue at https://github.com/pytorch/pytorch/issues Nodes involved in the conflict: z [For PyTorch developers: one of the above constraints is wrong. Either the node should be recomputable, saveable, or not required for backward.] [Debug: min-cut path] z-Failed to compute min-cut on following graph:rScldjjjjSNr-join readwriteedgelistgenerate_edgelistnxrsrJrqzsolve_min_cut..d % ",,"7"7"I"I("STrLcldjjjjSr=r>rCsrJrqzsolve_min_cut..n rErLc3,K|] }||f ywrFr])rmrnrs rJroz solve_min_cut..v s8Q $8sc30K|] }|vs|fywrFr])rmr non_reachableus rJroz solve_min_cut..w sAa=.@q!fAs  z node_in[:-3]=z != node_out[:-4]=c3(K|] }| ywrFr]rmrI name_to_nodes rJroz solve_min_cut.. s2d 2sc|SrFr])rnode_idxs rJrqzsolve_min_cut.. s (1+rLrr)rrVrrVrWr;rU)rIrVrW str | None)rIrVrirbrWztuple[float, str | None])r)rIrVrrrWr;)r!r`r"rrWr)r4rrWr)Kr$get_default_op_listr<rrDr=rsnetworkx ImportErrorrDiGraphrrdrerrrurvrr5r4ryrrrrArfloatrrrrr1rir rrrvrrrCrrrr rr minimum_cutNetworkXUnboundedr;rhandlers owning_modulerrr_get_unique_pathopenwrite Exceptionr?r@rArB_find_infinite_capacity_pathr setdefaultrPvisualize_min_cut_graphrrrget_name_to_noder|rt)Mrr;rrjoint_module_ops ops_ignorederrrrI ban_reasonis_non_tensor_nodeweightcannot_save_reasonrr% used_nodeordersfw_usersfirst_unfusible_usevisited start_nodefusible start_orderr6r cut_value partition unbounded_excstructured_tracing_enabled fx_graph_filer finf_pathnode_constraintsraw_path_nodesr6 from_nodeto_noderbase from_baseto_baseconstraint_linesr4 constraintscconstraints_str raw_path_strsvg_pathlocal_files_msg tlparse_msg reachablecutsetnbrs cut_nodesnode_innode_outr0r)rrrr1r)rKrrrOrQrIrDrrr;rJsM ``` @@@@@@@@@@@@@@@@rJ solve_min_cutrsb <"$H%& #))&   '4 $554 *   9;G( A& 9vF &$&$4G&$ !&$Przz|H(2 L@!!Y 77h   9.. .9555!!II&!XXX "!!II%!XXP "  $     E!D    d  (~ >  & (/A B .d3  # #D )j (z :  " E}DII'EUtyy SDIIe4Dell)S%S  t =./F!%   e$}6F&G%%)"%8")8i;;* &F& 6TXX#579J    E! F"&'9&:;     dii%/V1Cf  UJJ D    F" E!(   gYX4,,"44 ;I&OO++D1&&t,F "+I4L4LT4RH6{Q&:8S[&Q#!)//2;D!006%2248;NN&y$7</$O%%229=/ %2248 5T:!; ;P11'1|%++! VJ++J7'' 3Z@2G$00 C **3/+2CCG )HH."!..s3!..z: 15IIVD!006&sD1 4w1G1G1Mt0TU V)g,"! VFb-r~~h&I 9F )I}*4,F8i8B4 AdAAB",I#! 3B<8CR= (  ~-?" O CRL  i !$K0L+4[5F5F+GHic4c HH2 28ML  %%W  O  N  R   V%))*<*<%="%) #' "00  6  ++!&tD,%  0  --CVLMmS) &Q % & & & 6015M 6 ",,"7"7"I"I("ST  -  09 68 &ZN ! /7 * 7F%%g.((1D$//b9@@H&(3D$//b9@@H"9-w1GG(3D$//b9@@H!.i 8I+G4G$//<CC%i[1% .+- *:*@*@*B :& ; ''"YKq(9:$:A$++fQCL9: : #ii(89O!;;~6L%` `&`" `&%j & `?/`:5j :`??I j  Akctdg}tdgfg}|r|j\}}|j|D]}||vr|||}|j dd}|t j k(s |tk(s?|j dd}|||f} || gz} |dk(r| cS|j||j|| f|ry)zBFS from source to sink following only infinite-capacity edges. Returns a list of (from_node, to_node, reason) tuples representing the path, or None if no such path exists. rrrrunknownrN) r$rpopleft successorsrrurvr rr) rroqueuerI edge_pathneighbor edge_datarrnew_edgenew_paths rJr`r` s($G>$  G ''..I;ay < =1  ''..I;ay < =['9+ ..rLcddl} ddl}|j j |j}|j|d}|jD]c}||j|jd}|jt||tdk(sS|jd e|j!j#d }t%d d }t'|d 5} | j)|ddd||fS#t$rtj ddYywxYw#1swY||fSxYw)zVisualize the min-cut graph to an SVG file. Returns (path_to_svg, svg_content) tuple. Both are None if pydot is unavailable. rNzMInstall pydot to visualize the min-cut graph for debugging: pip install pydotT)exc_info)NNrrvredutf-8min_cut_failed.svgr+)rTpydotrUr=rsnx_pydotto_pydot to_stringgraph_from_dot_data get_edges get_sourceget_destination set_labelrrW set_color create_svgdecoder\r]r^) rrDr dot_format dot_graphedgerir;rrxs rJrbrb s@%%h/99;J))*5a8I##%"$//+,T-A-A-CDZP s6{# U5\ ! NN5 ! "&&(//8K 0&9H h    [  3   [   , [  sD4D8 D54D58Ec xgtjtjtjtjtj tj tjtjtjtjtjtjtjtjtjtj tj"tj$tj&tj(tj*tj,tj.tj0tj2tj4tj6tj8tj:tj<tj>tj@tjBtjDtjFtjHtjJtjLtjNtjPtjRtjTtjVtjXtjZtj\tj^tj`tjbtjdtjftjhtjjtjltjntjptjrtjttjvtjxtjztj|tj~tjtjtjtjtjtjtjtjtjtjtjtj}tjtjtjg}|tjtjtjtjtjtjtjtjtjg z }|}|gtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjz }|tjtjgz }||z }|tz }|tjgz }|tDcgc] }t|c}z }t|}ttdtftjtjtjg}tjtjtjtjtjtjtjtjtj tj tjg }||z}t|t||t||Scc}w)N.)rrrratan2r@rCrDpow remainderfmod__and____or____xor__ __lshift__ __rshift__eqnegegtleltr9 bitwise_notceilfloorfracnegreluroundsilutruncr=log10log1plog2lgammaexpexpm1erferfccosacoscoshsinasinsinhtanatantanhatanhsqrtrsqrtr?sigmoidsoftplus thresholdthreshold_backwardclampwherelerpaddcmulgelu gelu_backwardrOmean_grad_sum_to_size sum_to_sizer:totype_asrrsqueeze unsqueezersub_to_copyaliasrslicetr;broadcast_in_dimexpand as_stridedpermuteselectrer<clone full_likevarstd _unsafe_viewrbroadcast_tensors scalar_tensorones new_zerosrarangetriuvar_meanisinfrfullzerosempty empty_likeargmaxmaximumiota'_low_memory_max_pool_offsets_to_indicesrgatherrc zeros_likerrr$rr native_dropout rand_like randn_likemm convolutionconvolution_backwardbmmaddmm#_scaled_dot_product_flash_attention'_scaled_dot_product_efficient_attention_flash_attention_forward_efficient_attention_forwardupsample_bilinear2d _scaled_mmr?)default_recomputable_opsrecomputable_view_opsrCmrDrBrAr@s rJrSrS sL: L: L: L: L:  L:  L: L: L: L: L: L: L: L: L: L: !L:" #L:$ %L:& 'L:( )L:* +L:, -L:. /L:0 1L:2 3L:4 5L:6 7L:8 9L:: ;L:< =L:> ?L:@ AL:B CL:D EL:F GL:H IL:J KL:L ML:N OL:P QL:R SL:T UL:V WL:X YL:Z [L:\ ]L:^ _L:` aL:b cL:d eL:f gL:h iL:j kL:l mL:n oL:p qL:r sL:t uL:v wL:x yL:z {L:| }L:~ L:@ AL:B CL:D EL:F GL:H IL:J KL:L ML:N OL:P QL:R SL:T UL:V WL:Z"\\4>>4::F           %H$! $! ""$! $! $!  $!  $!  $! $! $! $! $! $! $! $! $! !$!" #$!$ %$!& '$!( )$!* +$!, -$!. /$!0 1$!2 3$!4 5$!6 7$!8 9$!: ;$!< =$!> ?$!@ A$!B C$!D  E$!F 55G$!$LT[[ 99( / ! N1!3A!6 NN!":;HS#X./  dnndoo>J   !!   00 44 %% ))     #Z/K ()8  +!Os7d7cJi}|jD]}|||j<|SrF)rr)rrOrIs rJrcrc s.')L '"& TYY' rLctj}|dk(r t|||S|dk(r t|||S|dk(r t |||S|dk(r t |||S|dk(rZt jdtj||||}t ||t|jt| St|tr||||||\}} d || fStd |) Ngreedyilpdpr)dynamic_memory_budget_dpzdynamic_memory_budget_dp is an experimental solver. It does not guarantee performance improvements. Additionally, it is not guaranteed to be stable.)rall_recomputable_banned_nodes recorded_knapsack_input_memories recorded_knapsack_input_runtimes)graph_info_provider) knapsack_algomax_mem_budgetrz,Not aware of memory budget knapsack solver: )ractivation_memory_budget_solverr*r+r(r)r=warningr'inialize_from_graphr,get_knee_point_memory_budgetrrr) rmemoryruntimes max_memoryr;r,SOLVERr/saved_node_idxrecomp_node_idxs rJ#_optimize_runtime_with_given_memoryr< s# 3 3F vx<< 5FHj99 468Z88 3 3-fh KK - -  ? 0CC#*G-3-5      $7 **))+   F0 1*0 KY8U+ '^_55I&RSSrL no_dispatchct|j}dfd }|Dcgc] }|| }}|jDcgc] }|| }}|j||Scc}wcc}w)Nct|S)NrF)r )drGs rJrealize_symbolz8_remove_symbols_without_guarding..realize_symbol s X66rL)stride)rAztorch.SymInt | intrWr)rshaperCnew_empty_strided)rrGrDrBrrCs ` rJ _remove_symbols_without_guardingrF sk ME7). .1^A  .E .)* 4AnQ 4F 4  uV  44 / 4s A(A-cx tj}d d}|dk(ry|dk(rat5ddlm}t j |jjf\ |j fd}|cdddS|dk(rudd l m }t j |jjf\ |d 5}ji dddj}t|dSt|t r|St#d |#1swYyxYw#1swYWxYw)Nczt|tjrAt|jdtj rt |jddSt|tjrAt|jdtjrt|jddSt|tjr(t|jdtjryt|tjr(t|jdtjry|S)NrrErF?T) rrrrrrArFrr rrrHs rJmaterialize_argz)estimate_runtime..materialize_arg s a !j &M3AFF5MDQ Q 277 # 166%=%,,(O$QVVE]TB B 277 # 166%=%..(Q 277 # 166%=%--(PHrLtestingr&profiler) benchmarkerc(jiSrF)r)rrrIsrJrqz"estimate_runtime.. s;4;;3O3OrLflops)FlopCounterModeF)displayz Not aware of runtime estimator: )rr rWr )r*activation_memory_budget_runtime_estimatorr>$torch._inductor.runtime.benchmarkingrMrtree_maprr benchmark_gputorch.utils.flop_counterrPrget_total_flopsrCrrr) rI RUNTIME_MODErJrMmsrPmode counted_flopsrrs ` @@rJestimate_runtimer\ s'DDL y   " ]  H!???TYY .estimate_activations_size" s3x./#55rLc|dz z z Sr_r])sz max_act_size min_act_sizes rJget_normalized_sizez4choose_saved_values_set..get_normalized_size+ sS\L899rLc&|z z z SrFr]) activationsrbrerfs rJ get_mem_ratioz.choose_saved_values_set..get_mem_ratio. s")+6E < '  rL)rrr)r)get_node_storagec3.K|] }|ywrFr])rmrIrks rJroz*choose_saved_values_set..L sT4 0 6TrKc|Dcgc]1}|jtdkr|vs t|r|3c}Scc}wr_)rrr)rr1rkinput_storagess rJget_recomputable_banned_nodesz>choose_saved_values_set..get_recomputable_banned_nodesN sL "  S)$Q'~=03    s6?rTrTr=c 5t| t|d| \}}}dddt}D]} |j ||j s t dt|| |\}} tr+t| | D cgc] } t| c} | |fS#1swYxYw#t$rYwxYwcc} w)Nrz:dont_ban must be a subset of all_recomputable_banned_nodes) rr,saved_node_idxsrecomputable_node_idxsexpected_runtimememories_banned_nodes normalized_memories_banned_nodesruntimes_banned_nodesmin_cut_saved_values) r<rCr$r BaseExceptionissubsetrrr<r rQ) memory_budgetr;rrsrqrrrr0r)r6r1aggressive_optionsr,rtr>rvs rJget_saved_values_knapsackz:choose_saved_values_set..get_saved_values_knapsacky s!]  4%%M1%-    & )3 ) C  :3?@   !>? L (       a ! 4'.K /'=!1)F'$%HQK'2G&;%1 ---]  $!  *'s#B8CC8C CCcN|\}}|t|z |fS)N)r;r)rO)rr)rsrjr|rr;rvs rJestimate_for_budgetz4choose_saved_values_set..estimate_for_budget s@-FYK. *L*)*-==l+ rLrrIgMbP?r ) )figsizeo)markerz.4fz offset points)rrcenter) textcoordsxytexthaz Memory Budgetz Runtime of Recomputed Componentsz:Pareto Frontier of Memory Budget vs. Recomputation Runtime)exist_okr_rank_memory_budget_paretor6rz%Generated Pareto frontier curve at %s)rzr;r)r)r`rWrW)rdrWrWrW)rir`rWrW)rrbrWr`)rzrWr;r_rfx.GraphrWztuple[list[fx.Node], float])rrWrWztuple[float, float, float])8rrrban_recompute_used_far_apart!ban_recompute_long_fusible_chains#ban_recompute_materialized_backwardban_recompute_not_in_allowlistban_recompute_reductionsaggressive_recomputationrrartorch._inductor.fx_utilsrkr$rrr%r.rtrQrr\torch.utils._mode_utilsr>visualize_memory_budget_paretorrsortmatplotlib.pyplotpyplotfigureplotr|annotatexlabelylabeltitlegridgcfshowrgetcwdmemory_budget_pareto_dirmakedirsrrris_initializedget_rankrr?r2savefigr=r3)+rr;rzrruntime_optimized_saved_valuesr6rgmore_aggressive_optionsmore_aggressive_saved_values%aggressive_recomputation_saved_valuesrrorecomputable_banned_nodesr1must_save_nodesrIr~optionsbisectslhsrhsmidpltitemx_valuesy_valuestxtfigfig_dir rank_suffixfig_namer{r,rbrjrkr|rnrertrfr>rvs+`` @@@@@@@@@@@@rJchoose_saved_values_setr s qMA-XYfXg h  $$AA#)#K#K%+%O%O & E E88 O&&! "'',).$)  (5)%"A --6-Y-=-=>L,-KLL|#--: &##(%*  '4Y 7'# !12]B++  %;HY 2;7)<:;mK449T9CSCSTTN )  !>l K+ 66::k5 )-=-G-G G O -!0H!!%+!x%! ()Q./112O-.HQK(,I#'41.1.)11.@H1. $1.1.f,,  's+-@-EF 1:ab>WQZ^ + GAJ/0G";;=Sq6CF?T)NN3'NN3')3q6CF?a*?@qr7c!"g%NNC:.qr7c!"g%NNC:.  '(/0DG00(/0DG00 7 # 8C0 ) FAs LLs)hqk"*    ?# 56 NO ggi  ))+  * * 655G KK$ /    ) ) +0A0A0P0P0R"5#4#4#=#=#?"@AK77<< +K=:L:N9OtT  H ;XF %#yk   m !n10s* 6T7 T<T< U(U U ! Ucddlmdd}dfd }tjj rctjj rDtjj dkDr"||r||rt55|Dcgc]}|jc}g}ttjj Dcgc]}g}}tjj||dt|}g} i} t|D]w\} } | D cgc]} ||  }} d}|D]C}t|}||z }| tjjk(s5|| |j<E|| d<| j|ytj | tjj"j%}tjj'|tjj"j(j*t-tj.|j1}d |d | t3d d fd ||Dcgc]}|| }}dddddd|S|Scc}wcc}wcc} wcc}w#1swY)xYw#1swY|SxYw)Nr)unset_fake_temporarilyc|jD]K}t|jtjj s2|jj dvsKyy)N>c10d_functionalrTF)rrrrrrr)rrIs rJhas_collectivesz3_sync_decision_cross_ranks..has_collectives sM%% D UZZ22++''+RR   rLcdjd|jD}tj|j dj }t tjjDcgc]}dc}t55tjj|ddddddtfdDScc}w#1swY*xYw#1swY.xYw)N/c34K|]}|jywrFr)rmrs rJrozE_sync_decision_cross_ranks..has_same_nodes.. s>qAFF>rrc3.K|] }d|k(ywrr])rmr all_inputss rJrozE_sync_decision_cross_ranks..has_same_nodes.. s:!:a=A%:rK) r?rhashlibsha256encode hexdigestr.rrget_world_sizer>all_gather_objectr)rnode_strrar6rrs @rJhas_same_nodesz2_sync_decision_cross_ranks..has_same_nodes s 88>K,=,=>> 89CCE$)%*;*;*J*J*L$MNqdN ] D24 D    / / F C D D:z::: O D D D Ds* C#C4!C(?C4(C1 -C44C=r&z total size)rrzpicked_rank_idx=z, saved_nodes of current rank=rc dddS)N)aot_joint_graph_sync_decision_cross_ranksrrr]r]rLrJrqz,_sync_decision_cross_ranks..= sG (%rLcSrFr])sync_decision_cross_ranks_strsrJrqz,_sync_decision_cross_ranks..A s#@rLr)rtorch.fx.GraphrWr;)torch._subclasses.fake_tensorrrrrrrr>rr.rrcr|rQrrr[distributed_c10d_get_object_coll_device all_reduceReduceOpMAXrargminrr)rr)rrrobjectsr6saved_ops_names_all_ranksrO saved_sizessaved_ops_with_sizesr0saved_ops_namesop_name saved_nodes saved_sizerI size_of_nodesaved_sizes_tensorpicked_rank_idxrnrrs @@rJr0r0 sE ; &&(    , , .    , , . 2 K ( ; ' ]* 24* (45156G!%"3"3"B"B"DE:: %:    / /0I7ST: V+K8L%'K35 (12K(L /$_DST|G4T T 'GD#+D>L,.Je//88:::F,TYY7 G 6@$\2"":. /"'((99QQS"     ( ("u'8'8'I'I'R'R'V'V ) "%,,/A"B"G"G"IJO.>>OOmoCnD-E ) A  *C?)S$% QLQ* * X <W6:U:Q* * * X sbJ7J+J/-J+ J%A J+. J! ::J+5DJ+5 J&J+J7J++J4 0J77Kcd}|rdnd}tt|jjd}|jjdtj j jD]}t||jdj}t|tjsBg}|jjdD]}||jvs|jj|5|jj!|d|} |d z }|j"d | j"d <| }|j%| d d d |s|jj|5|jj'dtj j jg|j|i} |j)| d d d d |j"j+d} | r8| \} } g| |Dcgc]}|j"d c}}|| f j"d<|jj-||S#1swYxYw#1swYxYwcc}w)u Graph-safe RNG lets torch.compile use CUDA Graphs for graphs with RNG ops. For graphs without HOPs, the partitioner adds placeholder nodes fwd_rng_state_* and bw_rng_state_* to the forward and backward graphs. At runtime, the AOTDispatcher retrieves these RNG states and passes them to the compiled graphs. This works well for no-HOP graphs. With HOPs, the partitioner runs recursively: it first partitions the HOP (producing forward/backward HOP subgraphs) and then stitches them back into the outer joint graph. For HOPs that contain RNG ops, the outer joint graph now includes HOP subgraph modules with extra RNG placeholders. We must thread these placeholders through the outer module partitioned forward and backward graphs—this function does exactly that. It collects the RNG placeholder nodes from the HOPs and creates corresponding placeholders in the outer forward and backward graphs. There is a catch: for a short period, the joint graph is in a “bad” state. The HOP subgraphs expect additional inputs (because of the new placeholders), but the outer graph call sites don't yet provide them. We can't fix this in the joint graph because the joint graph's input signature is fixed (primals, tangents). As a compromise, we keep the joint graph in somewhat of a bad state for some time and, once the outer forward and backward graphs are partitioned, insert the corresponding RNG placeholders and wire up the calls. rrrrrr)rrr6r&rNT)propagate_metaeager_input_vals)rrrrrrrinvoke_subgraphrrrrrrwrr8rrrrrrr)modulerr rng_string last_inputhop_nodernew_rng_inputsplaceholder_nodernew_hop_node_with_fixed_args eager_vals eager_args eager_kwargsinpnew_eager_argss rJr4r4K sf<I$/_Jhv||66-6HIJJLL++ 599#9#9#I#I,,268==#3#:#:; h /,.N$,NN$=$=$=$O 9 !1!6!66 55jA9$*LL$<$<)l!I;7% "Q 0@0E0Ee0L u-%. &--i899 9\\11(; 39<<3K3K' ..>>9(--9.9 40 224T3 &]]../AB /9,J &#&5CDc#((5/D&N '$M0556HI ''1Y,2\ MI99  "Es AI A#I-I9 I* -I6 c t|jt|jjD]m}|jdk(r d|j vrj |nt|rj ||vsSj|jottt|jj}ttt|jj}||z}t||\}}} } j} jd|Dt!|j||| d} tfd| jDtfd|jjD} tfdt#|D}d }i}|jjD]}|vs|||<|d z }t%|| | ||S) Nrtangentsrc3FK|]}||jdk7s|yw)Nrr)rmrs rJroz!classify_nodes.. s$!-ADDH4Ds !!!rc3ZK|]"}|jdk7r|j$ywrrrNs rJroz!classify_nodes.. s.8  77h  TYY8s(+c32K|]}|vr|vr|ywrFr])rmrIrdrvs rJroz!classify_nodes.. s*6  ( (T9J-J 6sc32K|]\}}|vs |ywrFr])rmr1prs rJroz!classify_nodes.. s"-a!7T2T-s rr&)rcrr$rrrrrrrrrr5r4r&rrr|r_)r rr!rIrrrar"r#r$r%reforward_only_graphrfrifw_cntrhrOrdrvs ` @@@rJr&r& s $L$6$67L-7\""((1 77m # dkk(A  ! !$ ' !$ '  ! !$ ' $ $  $ $TZZ 01 L,>,>,D,DEFM!&)P>P>V>V"WX 3 3F OCK/1B)--/<FK1BI.88&,,8. ,66 &&,,6,O #-- .-#FH""(( $ $#HTN aKF # rL)rc  |jj|j|j}tjrt |}||_|j}t |}t|} |r t|d}tjs t|t|t||g}t|||} t| jdk(rt!||||| j"St%|jj&D]} | j(dk(rt+d| _#| j/| sd| _}|jtjjjj u@ywrF)rrrrr(r)rmrJs rJrozWmin_cut_rematerialization_partition.._is_assert_only_symbool..Gs-W!AHH  = = E EEWsAA)rrrrrrrrrns rJ_is_assert_only_symboolzDmin_cut_rematerialization_partition.._is_assert_only_symboolCsP qvvzz%(%-- 8 XAGG q  XWqwwWW rLc0t|xr | SrFr)rnrs rJrqz5min_cut_rematerialization_partition..Lsk!nG-DQ-G)GrLc6t| xr t| SrF)rr1rs rJrqz5min_cut_rematerialization_partition..Qs[^+EN14E0ErLrrrTc32K|]}t|ywrF)rQrs rJroz6min_cut_rematerialization_partition..s'J 'Jrz'Theoretical Activations Stored: %.2f GBz,Theoretical Per Activation Storage Sizes: %sc3TK|] }|jdk(s|j"ywrNrrs rJroz6min_cut_rematerialization_partition..$% 477o;UDII% rc3TK|] }|jdk(s|j"ywrrrs rJroz6min_cut_rematerialization_partition..rrrz# remat/fw/bw: %d/%d/%drTzCount of Ops Rematerialized: %sr);rrrrcser8rrr!r"r#r$r%r&rrdr?rirrrrrryrrDactivation_memory_budgetrrrrWrr0rrr1rr1r2rr3r:r4r<rtrQrrOr=rsr$rrrrrrPrrW)r r5compilerr!rr cse_graphrr9r:r;rIrrzr)rrr=r>rr1 sorted_sizestotal_activations_size_gbfw_module_nodesbw_module_nodes remat_nodescountsrematerialized_opsrs @rJr r  srD**,   Dzz & & $$K!5l!C%=l%K"!-lQVW  : :|,\*|,$,(*%3_I 9 & &'1,  +*G(1(M(M   ++112R 77h  #CD ))$/ !D  #CD   R$'(9(94;L;Lq;P$Q! RR33M!! diimmOT:E B IIo6M +#L ((1+|L   G O f^\BCE|TL 4'-'$-$I$IIy" )#8iC4H$ Iy4I>I**  %     1 1  y)Iy)I.yeLI.ydKIlK SV4KL %('J\'J$JS$P! :DyyK'GDKKAR,Ss4;;6678A=8 >  %      $ LLN 3 3A 6  24FG i =Ls(Sc|rWtj|j}tj||}|jj D] }i|_ tjj|\} } | sdtjz} tjd| | tj||||} | j!} t#| d| j%dz} | | }| | |y| ||y)NrbzWriting FX graph to file: %s%s)parse_stack_tracedot_graph_shapewrite_)prog)rdeepcopyrrrwrrrrsplitextrtorch_compile_graph_formatr=rsr# FxGraphDrawerget_main_dot_graphrlstrip)tracedfnamefigname clear_metarrrrrIr~extgr write_methods rJ draw_graphr'sMM&,,/  2LL&& DDI   'ID# F555HH -tS9""+'  A A1hC89LfSENE |UU&rLrU)rrrWr;)rIrVrWr)rrVrWfx.Node | None)rIrVrWr()rIrVrzdict[fx.Node, Any]rWr()NF)rrrar`rr`rzlist[AOTOutput]rrRrr;rWr)r rr!rrWzEtuple[list[fx.Node], list[fx.Node], list[AOTOutput], list[AOTOutput]])r)r`rrrWrt)r/z#list[fx.Node] | tuple[fx.Node, ...]rWr)g@rxr) rrrI torch.fx.NoderCrWrDrWrErrWr))rrrIr)rLr)rQ torch.dtyper>rWrPrWrErrWr))r[rrWrW)rWzlist[torch.dtype])rIr)rWr;)rWr*)rfr*rWztuple[float, float])r)rrr!rrWrt)rrrWrt) rrrrrrr!rrWrt)Nr) r)r`rrrrriOrderedSet[fx.Node] | Noner!rrWrtrF)r rr)r`rr`rzlist[fx.Node] | Noner!rrir+rr;rr;rW%tuple[fx.GraphModule, fx.GraphModule]) r rr5r r!rrlist[int] | Nonerir+rWr,)rYrrfr*rWr)rrrWrt)rWz!list[torch._ops.OpOverloadPacket])rztuple[Any, ...]rergrWzlist[tuple[fx.Node, int]])rxrrWr)r=torch.fx.GraphModuler>r.rr)rr)rz torch.devicerrrr)rr)rWz#tuple[torch.fx.Node, torch.fx.Node]) r rr=rr>rrrrWr,)r rrWrt)r rrr;rWr) rrr;r_rrrr+rWz)tuple[list[fx.Node], OrderedSet[fx.Node]])rnx.DiGraph[str, dict[str, Any]]rWz!list[tuple[str, str, str]] | None)rrrrrWr)rr/rWztuple[str | None, str | None])rWr?)rrrWr)rrr6 list[float]r7r0r8rWr;r_r,r`rWz"tuple[float, list[int], list[int]])rrrGrrWr)rIrVrWrW)r&)rrr;r_rzrWrWr`)rrr)list[torch.fx.Node]rWr1)rrrr;rWr)r rrz list[int]r!rrWr_)ry) r rr5r r rr!rrr-rWr,)fx_graphTNFN)r r.r!rr"rr#r;rzstr | list[str] | Nonerr;rrRrWrt) __future__rrrrrrloggingrurros.pathrrrVrrcollections.abcr dataclassesrrtypingr r rtorch._inductor.inductor_primstorch.distributedtorch.fxrtorch.utils._pytreeutils_pytreertorch._dynamo.utilsr r ;torch._functorch._activation_checkpointing.ac_logging_utilsr $torch._functorch._aot_autograd.utilsrtorch._inductorrr!torch._inductor.custom_graph_passrr"torch._library.fake_class_registryrtorch._library.utilsrtorch._loggingrrtorch._logging._internalrrr%torch.fx.experimental._backward_stater"torch.fx.experimental.proxy_tensorrrtorch.fx.experimental.sym_noderr%torch.fx.experimental.symbolic_shapesrrrr r!r"torch.fx.passesr#torch.utils._ordered_setr$torch.utils.checkpointr%r-_activation_checkpointing.graph_info_providerr'"_activation_checkpointing.knapsackr(r)r*r+,_activation_checkpointing.knapsack_evaluatorr,_aot_autograd.descriptorsr-r.r/_aot_autograd.functional_utilsr0_aot_autograd.graph_compiler1_aot_autograd.logging_utilsr2_aot_autograd.utilsr3r4r5r6r7 compile_utilsr8r9r:rTrDsympydebug_partitionerr<r\ getLoggerrXr=rrr;r?r_rrrrrrrrrrrrrrrr r rrrr&r+r2rMrWr]rgrnrprtrrrrrr?rr rBrQrYcachercrhr2rr1r#r$r%rr!rr`r\rbrSrcr<rr>rFr\rr0r4r&r r'r]rLrJr\s "     *$*% %$$<A5@+7.A?HL)/3L L A7;IH%66t6'g''1^1 yy~~  >> >2     B    I 2 -  H $) rr rr# r  r " rrjXCBJK 5J J69JJJ$; K K K K  K  K  K\C! C! C!C! C!  C!  C!C!C!L9"G45 FGRMGh XXX*X X  X~?C ' ' ' ' "< '  '  ' \04 k"?C$)&+k" k"k"#k"- k"  k""<k""k" $k"+k"f7;>B b  b b  b $4 b "< b +b J c(":R "P P&8PPK\[*#[*#[*[* [*  [*  [*"[*"[*)[*|a  a a a  a + a H @'<.K5 5;?55x,0 v &v &v &#v &) v & / v &r-&B /$!-$!"$!NeP-T-T -T-T -T  -T $1 -T(-T`05*N`z z z z  z zNN/BNNbO O)-OOd> >#,>> >H| 7; |  | | |  | $4 | +| D#'#"&' ' '' ' ! '  ' ' 'rL