Lj1mUddlZddlZddlZddlZddlZddlZddlZddlZddl Zddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlmZddlmZmZddlmZmZddlmZmZmZddlmZddlm Z ddl!Z"ddl#m$Z$dd l%m&Z&dd l'm(Z(ed Z)ejTe+Z,ejTd Z-ej\Z/d Z0dZ1dZ2dZ3dZ4dZ5da6ede7d<da8dZ9eGddZ:eGddZ;e:Z<e;a=ej|ej|ej|ej|dddddej|d Z?dddddddddddddddddddddddddddddddddddddddddddddddddddddd5de@dzde@dzd e@dzd!e@dzd"e@dzd#e@dzd$e@dzd%e@dzd&e@dzd'e@dzd(e@dzd)e@dzd*eAd+eAd,eAd-eAd.eAd/eAd0eAd1eAd2eAd3eAd4eAd5eAd6eAd7eAd8eAd9eAd:eAd;eAdeAd?eAd@eAdAeAdBeAdCeAdDeAdEe@dzdFeBeCe@eAzfdzdGeAdHeAdIeAdJeAdKeAdLeAdMeAdNeAdOeAdPeAdQeAdReAdSdfldTZDdSeEejfdUZGddVZH d ddWZIdSejfdXZJdYZKdZZLd[d\je jeKe jeLgzd]zZOdd^ZPd_ZQd`ZRddaZSddbZTdceCdSeCfddZUejdeZWdfZXdgeCdSeeCdzfdhZYddiZZdSeAfdjZ[ejdkeCdle\eCdmfdSeCfdnZ]dkeCdSeCfdoZ^GdpdqejZ`drZaeaZbddsZce ZddtZeduZfddvZgddwZhdxZiddyZjdddzZkGd{dejZmdd|Znejdd}ZpddSeAfd~Zq ddSeAfdZrejdfdZtGddee)ZueevaweBeCevfe7d<devdSdfdZxdSevdzfdZyddfdeCdeCdegeCezzdzfde$dzdSdf dZ{eBfdddddddeCdegeBeCefe\eCe@fzfdegeCezzdzfdeAdeAdeAde$dzdSdfdZ|eBfddddddeCdegeBeCefe\eCe@fzfdegeCezzdzfdeAdeAdeAdSdfdZ}ddl#Z"ddl%Z"ddl~mZy)N) defaultdict)CallableSequence) dataclassfield)AnyGenericOptional) ParamSpec)WeakSet) CompileId)log_trace_structured_event)CapturedTraceback_Pz torch.__trace TORCH_LOGSTORCH_LOGS_OUTTORCH_LOGS_FORMATTORCH_LOGS_TRACE_ID_FILTER TORCH_TRACE TORCH_DTRACELazyTraceHandlerLOG_TRACE_HANDLERFdedicated_log_torch_trace_ceZdZUeeZeeeefed<ee Z e eed<ee Z e eed<ee Z e eed<ee Z e eed<eeZeeefed<ee Ze eed<eeZeeej$fed <d Zd Zd eeezd dfdZ ddZddZddZd e efdZdZdZdZy) LogRegistrydefault_factorylog_alias_to_log_qnamesartifact_log_qnameschild_log_qnamesartifact_namesvisible_artifactsartifact_descriptionsoff_by_default_artifact_namesartifact_log_formattersc||jvSNr!selfnames W/var/www/html/ia-enhanced/venv/lib/python3.12/site-packages/torch/_logging/_internal.py is_artifactzLogRegistry.is_artifactet****c||jvSr')r)r*aliass r,is_logzLogRegistry.is_loghs4444r/ log_qnamesreturnNcHt|tr|g}||j|<yr') isinstancestrr)r*r1r3s r, register_logzLogRegistry.register_logls# j# &$J.8$$U+r/c|jj||r|jj|||j|<|r|jj||#t j ||j|<yyr')r!addr"r#r$logging Formatterr%)r*r+ descriptionvisibleoff_by_default log_formats r,register_artifact_namez"LogRegistry.register_artifact_namers~ %   " " & &t ,+6""4(   . . 2 24 8  !181B1B:1ND ( ( . "r/c:|jj|yr')rr:)r*artifact_log_qnames r,register_artifact_logz!LogRegistry.register_artifact_logs   $$%78r/c:|jj|yr')r r:)r* log_qnames r,register_child_logzLogRegistry.register_child_logs !!),r/cttjj|jj Sr')set itertoolschain from_iterablervaluesr*s r,get_log_qnameszLogRegistry.get_log_qnamess+9??001M1M1T1T1VWXXr/c,t|jSr')rIrrNs r,get_artifact_log_qnamesz#LogRegistry.get_artifact_log_qnamess4++,,r/c,t|jSr')rIr rNs r,get_child_log_qnamesz LogRegistry.get_child_log_qnamess4(())r/c||jvSr')r$)r*artifact_qnames r,is_off_by_defaultzLogRegistry.is_off_by_defaults!C!CCCr/r4N)__name__ __module__ __qualname__rdictrr7list__annotations__rIrr r!r"r#r$r%r;r<r-r2r8rArDrGrOrQrSrVr/r,rr?s, 5:$4OT#tCy.1O %*#$>S> "'s!;c#h; %S9NCH9#("<s3x<-2$,G4S>G /4C.H!3s8H=BRV`_ An artifact is a particular type of log message. Each artifact is assigned to a parent component. A component can emit many different kinds of artifacts. In general, an artifact is emitted if either its corresponding setting in the argument list below is turned on or if its parent component is set to a log level less than or equal to the log level of the artifact. Keyword args: all (:class:`Optional[int]`): The default log level for all components. Default: ``logging.WARN`` dynamo (:class:`Optional[int]`): The log level for the TorchDynamo component. Default: ``logging.WARN`` aot (:class:`Optional[int]`): The log level for the AOTAutograd component. Default: ``logging.WARN`` autograd (:class:`Optional[int]`): The log level for autograd. Default: ``logging.WARN`` inductor (:class:`Optional[int]`): The log level for the TorchInductor component. Default: ``logging.WARN`` dynamic (:class:`Optional[int]`): The log level for dynamic shapes. Default: ``logging.WARN`` distributed (:class:`Optional[int]`): Whether to log c10d communication operations and other debug info from PyTorch Distributed components. Default: ``logging.WARN`` c10d (:class:`Optional[int]`): Whether to log c10d communication operations related debug info in PyTorch Distributed components. Default: ``logging.WARN`` ddp (:class:`Optional[int]`): Whether to log debug info related to ``DistributedDataParallel``(DDP) from PyTorch Distributed components. Default: ``logging.WARN`` fsdp (:class:`Optional[int]`): Whether to log debug info related to ``FullyShardedDataParallel``(FSDP) in PyTorch Distributed components. Default: ``logging.WARN`` dtensor (:class:`Optional[int]`): Whether to log debug info related to ``DTensor``(DTensor) in PyTorch Distributed components. Default: ``logging.WARN`` onnx (:class:`Optional[int]`): The log level for the ONNX exporter component. Default: ``logging.WARN`` bytecode (:class:`bool`): Whether to emit the original and generated bytecode from TorchDynamo. Default: ``False`` aot_graphs (:class:`bool`): Whether to emit the graphs generated by AOTAutograd. Default: ``False`` aot_joint_graph (:class:`bool`): Whether to emit the joint forward-backward graph generated by AOTAutograd. Default: ``False`` ddp_graphs (:class:`bool`): Whether to emit graphs generated by DDPOptimizer. Default: ``False`` graph (:class:`bool`): Whether to emit the graph captured by TorchDynamo in tabular format. Default: ``False`` graph_code (:class:`bool`): Whether to emit the python source of the graph captured by TorchDynamo. Default: ``False`` graph_code_verbose (:class:`bool`): Whether to emit verbose/intermediate FX pass logs for graph code. Default: ``False`` graph_breaks (:class:`bool`): Whether to emit the graph breaks encountered by TorchDynamo. Default: ``False`` side_effects (:class:`bool`): Whether to emit side effects (mutations, hooks, etc.) that TorchDynamo codegenerates in the output graph. Default: ``False`` graph_sizes (:class:`bool`): Whether to emit tensor sizes of the graph captured by TorchDynamo. Default: ``False`` guards (:class:`bool`): Whether to emit the guards generated by TorchDynamo for each compiled function. Default: ``False`` recompiles (:class:`bool`): Whether to emit a guard failure reason and message every time TorchDynamo recompiles a function. Default: ``False`` recompiles_verbose (:class:`bool`): Whether to emit all guard failure reasons when TorchDynamo recompiles a function, even those that are not actually run. Default: ``False`` trace_source (:class:`bool`): Whether to emit when TorchDynamo begins tracing a new line. Default: ``False`` trace_call (:class:`bool`): Whether to emit detailed line location when TorchDynamo creates an FX node corresponding to function call. Python 3.11+ only. Default: ``False`` trace_bytecode (:class:`bool`): Whether to emit bytecode instructions and traced stack state as TorchDynamo traces bytecode. Default: ``False`` output_code (:class:`bool`): Whether to emit the TorchInductor output code on a per-graph basis. Default: ``False`` kernel_code (:class:`bool`): Whether to emit the TorchInductor output code on a per-kernel bases. Default: ``False`` schedule (:class:`bool`): Whether to emit the TorchInductor schedule. Default: ``False`` perf_hints (:class:`bool`): Whether to emit the TorchInductor perf hints. Default: ``False`` pre_grad_graphs (:class:`bool`): Whether to emit the graphs before inductor grad passes. Default: ``False`` post_grad_graphs (:class:`bool`): Whether to emit the graphs generated by after post grad passes. Default: ``False`` ir_pre_fusion (:class:`bool`): Whether to emit the graphs before inductor fusion passes. Default: ``False`` ir_post_fusion (:class:`bool`): Whether to emit the graphs after inductor fusion passes. Default: ``False`` onnx_diagnostics (:class:`bool`): Whether to emit the ONNX exporter diagnostics in logging. Default: ``False`` fusion (:class:`bool`): Whether to emit detailed Inductor fusion decisions. Default: ``False`` overlap (:class:`bool`): Whether to emit detailed Inductor compute/comm overlap decisions. Default: ``False`` sym_node (:class:`bool`): Whether to emit debug info for various SymNode opterations. Default: ``False`` export (:class:`Optional[int]`): The log level for export. Default: ``logging.WARN`` benchmarking (:class:`bool`): Whether to emit detailed Inductor benchmarking information. Default: ``False`` modules (dict): This argument provides an alternate way to specify the above log component and artifact settings, in the format of a keyword args dictionary given as a single argument. There are two cases where this is useful (1) if a new log component or artifact has been registered but a keyword argument for it has not been added to this function and (2) if the log level for an unregistered module needs to be set. This can be done by providing the fully-qualified module name as the key, with the log level as the value. Default: ``None`` cudagraph_static_inputs (:class:`bool`): Whether to emit debug info for cudagraph static input detection. Default: ``False`` autotuning (:class:`bool`): Autotuning choice logs, such as kernel source, perf, and tuning parameters. Default: ``False`` graph_region_expansion (:class:`bool`): Whether to emit the detailed steps of the duplicate graph region tracker expansion algorithm. Default: ``False`` inductor_metrics (:class:`bool`): Whether to estimate the runtimes of the nodes in a graph and log them to the metrics table. Default: ``False`` hierarchical_compile (:class:`bool`): Whether to emit debug info for hierarchical compilation. Default: ``False`` caching (:class:`bool`): Whether to emit detailed Inductor caching information. Default: ``False`` Example:: >>> # xdoctest: +SKIP >>> import logging # The following changes the "dynamo" component to emit DEBUG-level # logs, and to emit "graph_code" artifacts. >>> torch._logging.set_logs(dynamo=logging.DEBUG, graph_code=True) # The following enables the logs for a different module >>> torch._logging.set_logs(modules={"unregistered.module.name": logging.DEBUG}) zQUsing TORCH_LOGS environment variable for log settings, ignoring call to set_logsNctj|jjD]\}}| tj |r:t |t std|d||sCtj|Ytj|s|tjvr|tjvrHtd|d|ddjtjDcgc] }t|c}tj!tj"j%|||t'|rt)|xs|f}|D]m}t+|stj-||ntj/|tj!tj"j%|||otd|t1ycc}w)Nz!Expected bool to enable artifact z , received zUnrecognized log level for log z: z, valid level values are: ,z6Unrecognized log or artifact name passed to set_logs: )rJrKrk log_registryr-r6bool ValueError log_staterdr2r r; _levelToNamejoinr7rirget_is_valid_module_get_module_and_submodules_has_registered_parentr8rG _init_logs)kwargsr1valk found_modules module_namers r, _set_logszset_logs.._set_logss#//&,,.'--/J& JE3{''.!#t,$;E7+cUS--e4$$U+u 8U8U/Ug222$9%3%H #':N:N)OQ#a&)O PQS $$ 88<$11+{K$77 D(($<<@@'  !LUGTI& P  1*Ps/G-torchrorprzrqrxrrrrsr{r|r}rrr~rrrrtrurrvrwrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrWr^) LOG_ENV_VARosenvironlogwarningrrn)6ryrorprzrxrqr{r|r}rrr~rrrrrsrrrrtrurrvrwrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrs6 ` r,set_logsrswjbjj  _   OOmG)V555 5 5  5  555(55 55 555 !5"#5$.%5&"'5(")5* +5,-5./50.152"354556&758 95: ;5<=5>?5@(A5B*C5D$E5F&G5HI5J*K5LM5NO5PQ5RS5TU5V,W5X#r?r@s r,register_artifactres''k7NJr/c|tjvr$tdt|dt|d|d|z}t j |}||_tj|t||S)NzArtifact name: z. not registered,please call register_artifact(z") in torch._logging.registrations.z.__) rr!rreprr;rrcrDconfigure_artifact_log) module_qnamercrrs r,getArtifactLoggerrvsL777d=123--1--@,AAc e   S0 0E   E "C%C&&u-3 Jr/+-(|z?)ctj|jrd|_tj |jr'|j tjd|_yyNFT) rrVrc propagaterrfsetLevelr;DEBUG)rs r,rrsQ%%c&7&78 $$S%6%67 W]]# 8r/c,tjdS)Nz$((\+|-)?[\w\.]+,\s*)*(\+|-)?[\w\.]+?)recompiler^r/r,_gen_settings_regexrs ::= >>r/cBtjt|duSr')r fullmatchr)settingss r,_validate_settingsrs <<+-x 8 DDr/c rd d}|rtj}ntj}|rd}nd}dgttjj zt|Dcgc] }||dtj |"c}z}ddj|z}d}d |d |d |d }|Scc}w) Nczt||kDrtdt|d||d|t|z zzS)Nzstring length z exceeds max  )lenAssertionError)slengths r,pad_tozhelp_message..pad_tosA q6F? >#a&vh!OP P3&3q6/***r/zAll registered namesz?Visible registered names (use TORCH_LOGS='+help' for full list)ry z z a@ Examples: TORCH_LOGS="+dynamo,aot" will set the log level of TorchDynamo to logging.DEBUG and AOT to logging.INFO TORCH_LOGS="-dynamo,+inductor" will set the log level of TorchDynamo to logging.ERROR and TorchInductor to logging.DEBUG TORCH_LOGS="aot_graphs" will enable the aot_graphs artifact TORCH_LOGS="+dynamo,schedule" will enable set the log level of TorchDynamo to logging.DEBUG and enable the schedule artifact TORCH_LOGS="+some.random.module,schedule" will set the log level of some.random.module to logging.DEBUG and enable the schedule artifact TORCH_LOGS="+torch._functorch._aot_autograd" will set the log level of torch._functorch._aot_autograd and all its submodules to logging.DEBUG (directory-based logging) TORCH_LOGS_FORMAT="%(levelname)s: %(message)s" or any provided format string will set the output format Valid keys are "levelname", "message", "pathname", "levelno", "lineno", "filename" and "name". TORCH_LOGS_OUT=/tmp/output.txt will output the logs to /tmp/output.txt as well. This is useful when the output is long. z TORCH_LOGS Info z  ))rr!r"sortedrkeysr#r) verboserprinted_artifactsheadingr+lines setting_infoexamplesmsgs r, help_messagers+ (77(::(S  55::< = > . $<.<#E#Ed#K"LM    &++e,,LH8       C JU s"%B4 cdgttjjzttjz}dj t |}d|d|d}|S)Nryz, z Invalid log settings: z, must be a comma separated list of fully qualified module names, registered log names or registered artifact names. For more info on various settings, try TORCH_LOGS="help" Valid settings: r)r\rrrr!rr)rrvalid_settingsrs r,_invalid_settings_err_msgrsz  |3388: ; < |** + , YYvn56Nj!  C Jr/ env_var_strc`tjdk(}dtdtfd}|r||}|S)ae When we setup logging config as guide: https://docs.pytorch.org/docs/stable/logging.html Such as: TORCH_LOGS="+schedule,+inductor,+output_code" On Linux, it shows as: declare -x SSH_TTY="/dev/pts/0" declare -x TERM="xterm" declare -x TORCH_LOGS="+schedule,+inductor,+output_code" declare -x USER="xu" On Windows, it shows as: TORCHINDUCTOR_WINDOWS_TESTS=1 TORCH_LOGS="+schedule,+inductor,+output_code" UCRTVersion=10.0.22000.0 For Linux, it shows quotes by default, And Windows is not shows quotes. Besides that, Windows would auto assemble quotes when env var processing. On Linux, we will get variable: "+schedule,+inductor,+output_code" On Windows, we will get variable: '"+schedule,+inductor,+output_code"' So, we need remove the outer quotes for Windows. win32rr4clt|dk\r%|ddk(r|ddk(s|ddk(r |ddk(r|ddS|S)Nr"')r)rs r,remove_outer_quotesz?process_env_var_string_for_windows..remove_outer_quotes sF q6Q; qTS[QrUc\qts{qu|Qr7Nr/)sysplatformr7)r _IS_WINDOWSrs r,"process_env_var_string_for_windowsrs:0,,')Kss)+6 r/cJt|}|dk(riS|dk(rttd|dk(rttdt|stt |t j dd|}|jd}d }t}|D]}||\}}|d k(rd }tj|r3| td tj|}|j||^tj|r|j|t!|rat#|xs|f}|D]K}t%|stj'||ntj)||j||Mtt ||S) NhelpF)rz+helpTz\s+rc|jtd}|jtd}|dtk(rtj}||fS|dtk(rtj }||fStj }||fS)Nrr)replaceINCR_VERBOSITY_CHARDECR_VERBOSITY_CHARr;rERRORINFO)r+ clean_namelevels r,get_name_level_pairz0_parse_log_settings..get_name_level_pair+s\\"5r: ''(;R@ 7) )MME 5  !W+ +MME5  LLE5  r/ryrz#level must not be None for log name)rrrrrrsubsplitr`rr2rrrir-rdrrrr8rG) r log_namesrrr+rr3rrs r,_parse_log_settingsrs1(;H2~ 6e455 W d344 h '28<==vvfb(+Hs#I ! IB)$/ e 5=D   t $}$%JKK%==dCJ  U 3  % %d +  % %d + d #6t<GM, 9 -k: --k;G 33K@$$[%8  96x@A A/B2 r/cHtjj|}|duSr') importlibutil find_spec)rspecs r,rrVs! >> # #E *D t r/rc*tjj|}|y|g}|jatj|}t |dr@t j|j|dzdD]\}}}|j||S)ay Get a module and all its submodules (recursively). If qname is a package, this returns a list of all modules and submodules. If qname is a simple module, this returns a list containing just that module. Args: qname: The fully qualified module name Returns: A list of fully qualified module names, or None if the module doesn't exist N__path__.cyr'r^)xs r,z,_get_module_and_submodules..tr/)pathprefixonerror) rrrsubmodule_search_locations import_modulehasattrpkgutil walk_packagesrappend)rrrpackageimportermodnameispkgs r,rr[s >> # #E *D |gG &&2))%0 7J ',3,A,A%%s{&- (('5 w'  ( Nr/chtjjtd}| t |ayyr')rrrrrr) log_settings r,_update_log_state_from_envr/{s+**..d3K' 4 r/ctj|}tj}|jr(|j |vry|j}|jr(y)NTF)r;rrrOparentr+)rFcur_logregistered_log_qnamess r,rrsO *G(779 .. <<0 0.. .. r/abs_pathsys_path.ctj|j}|D] } |j|}t |cSt |S#t $rY9wxYw)a The string manipulation here is fairly expensive and we expect very high cache hit rates. Because `sys.path` changes very infrequently it's most performant to convert it into a tuple of strings and use that for the cache key because python has dedicated fast paths for list to tuple conversion and tuple hashing. (Empirically, a single top-level cache is about 2x faster than trying to cache individual parts.) )pathlibPathresolve relative_tor7r)r4r5abs_path_resolvedr rel_paths r,_make_module_path_relativer=sn X.668! !(44T:Hx= !  !!    sA A A cHt|ttjS)z Given an absolute filepath corresponding to a Python module which was loaded via normal import mechanisms using sys.path, convert it into a relative path relative to one of the Python search paths. )r=tuplerr )r4s r,make_module_path_relativer@s &hchh @@r/cFeZdZddddedeedzddffdZdZxZS) TorchLogsFormatterFN)tracetrace_id_filterrCrDr4c>t|||_||_yr')super__init__ _is_trace_trace_id_filter)r*rCrD __class__s r,rGzTorchLogsFormatter.__init__s  /r/c0 ttj|jdd}|3tj j |d}||j|S|j|_ |j|d|_ |j}|jrGddl m}|jxs|dk7}|r,|j s |j#|j|_|j r|dddk7r|dz}||j z}|j$r+|dddk7r|dz}||j'|j$z}d|_|j*sEt-j.r1t-j0rd t-j2d |_d|_|j*s6t6j8j:j=x} d |d |_d ddddd}|j |j>|j>} d|_ | d|d |_ tC|jD} |jFr'|j4jI|jFvry|j(| |jdtK|jLdzdd|jNd| d|jPd |j4|j@ |j*r|dk7rtSd| dtUjV|jXtZ} |j\4| dj_d|j\jadDz } | S|jad} dj_ fd| DS)Nrcz %m%d %H:%M:%Sr)configrtrrrz[rankz]:z []VIWEC)rr WARNINGr CRITICALz [__ri06dr:z%expected empty string for trace, got )defaultc3&K|] }d| yw)z Nr^).0ls r, z,TorchLogsFormatter.format..sLAtA3ZLsc3,K|] }d| yw)rNr^)rYrZr!s r,r[z,TorchLogsFormatter.format.. s<xq_QUV  $!-!E!E!I!It" "-)0088**,A NN ?? , & Q->2Q !&*&:&:6??&KFO ??v~HFOO#A   v~HD$$V%6%677A~~$"3"3"5$:M:M:O"' '8 ;F "]]99JJLL "(1-FN  (++F,<,N>NO "  $&*=/$;F !,V__=  ! !$$&d.C.CC  !*fnn-=Qs6<\\]^d^l^l]mmnj}}oQv~~.v/D/D.E G  >>Bw$'LQE%RSS(!DJJvEFGA~~)RWWL1E1Ed1KLLLHGGDME99tjD]+\}}tj|}|j|-t jD]C}tj|}ttj|5tfd|Et j!D]"}tj|}t#|$tjj t$d}|Vtjj dddk(r3ddlmcm}tj,j/|j0d}tjj t2dx}rda|}t6 t9|at:jtj<t?t6} | jAtCd t:jE| y) Nrc.tjSr')r; FileHandler) log_file_namesr,rz_init_logs..{s++M:r/TORCH_COMPILE_DEBUG01rtlparseT)rC)#rr/rrrLOG_OUT_ENV_VARrrOr;rrrrrlr StreamHandlerrQr TRACE_ENV_VARtorch._dynamo.config_dynamorLr rdebug_dir_rootDTRACE_ENV_VARGET_DTRACE_STRUCTUREDrrrrrrrBr) routrFrrrCtrace_dir_name dynamo_configdtrace_dir_nametrace_log_handlers ` r,rrWs M **..$ /C  "002%     * W^^$ %&99; 5 * U "002   *  ! !   $ :  +BBD$ 23s#$ ZZ^^M48N"**..1F"LPS"S44m&B&BIN**..>>> $( ,^<w}}%&'89""#5D#AB *+r/cBeZdZdZdedzddfdZddZdfd ZxZS) rzRLike FileHandler, but the file is allocated lazily only upon the first log messageroot_dirNr4c||_tjj|d|_t |_d|_y)NF)rr;HandlerrGstreamopen _builtin_open_pending_log_version)r*rs r,rGzLazyTraceHandler.__init__s4!   & !$)!r/c|j |jrA |j|j}d|_t|dr|j  t j j | |jy#|j}d|_t|dr|j wwxYw#t j j |wxYw#|jwxYw)Nclose)acquirerflushr%rr;rrelease)r*rs r,rzLazyTraceHandler.closes   2;;+ !%&* "673"LLN %%++D1 LLN"&&* "673"LLN4%%++D1 LLNs3 CB0C C*2CC!C''C**C<c|j|jd}ddlm}t |dr+t j dtjdntjjdstjdnxt jj|stjd|nBt j|t jstjd |n||_|jt j |jd d }t#j$r,t#j&rd t#j(d}t+j,ddt.|z|jd|_tjd|jj0t3j4d}|jd|jj0d |_nt8j;|y|jr.t<|}||j6rd|_tAyyy)Nz/logsr git_versionMAST_HPC_JOB_NAMEz>LazyTraceHandler: disabled because not fbcode or conda on mastzpytorch/trace:enablezYLazyTraceHandler: disabled because justknobs_check('pytorch/trace:enable') returned Falsez4LazyTraceHandler: disabled because %s does not existz6LazyTraceHandler: disabled because %s is not writeableT)exist_okrrank__zw+z.logF)modesuffixr!dirdeletezLazyTraceHandler: logging to %sztorch._inductorztlparse raw data: %s)!rr torch.versionversionr%rgetenvrinfor_utils_internaljustknobs_checkr existsaccessW_OKmakedirsrjrkrlrmtempfileNamedTemporaryFile LOG_PREFIXr+r;rrrrrFemit_log_torch_version)r*r~ TRACE_LOG_DIR torch_version ranksuffix inductor_logrJs r,rzLazyTraceHandler.emits ;; }}$ ' 5M=9 "56>HHX..>>?UVHHs 6HHN%="'':HHP% %2DM}}( DMMD9 $$&4+>+>+@#((9!;J&99!% 2   :DKK.s _&Ir/cSr'r^) version_infosr,rz$_log_torch_version..s<r/TF) metadata_fn payload_fnsuppress_contextexpect_trace_id)rtorch._environmentrtorch._utils_internalr __version__trace_structured)rrrrs @r,rrsD,>!,,*,;'L I' r/c(|j|i|y)a This function is similar to `logger.warning()`, but will emit the warning with the same message only once Note: The cache is for the function arguments, so 2 different callers using the same arguments will hit the cache. The assumption here is that all warning messages are unique across the code. If they aren't then need to switch to another type of cache that includes the caller frame information in the hashing function. N)r) logger_objargsrs r, warning_oncersJ''r/cdt|vS)NzThe .grad attribute of a Tensor)r7r`categoryfilenameryfilelines r,safe_grad_filterr "s ,CL @@r/c|tuSr') UserWarningrs r,user_warning_filterr&s ; &&r/cy)NTr^)rrs r,rr-rr/c#Ktjfd} |t_dt_y#t_wxYww)a A context manager that temporarily suppresses warnings, using public API: https://docs.python.org/3/library/warnings.html#warnings.showwarning. Useful to hide warnings without mutating warnings module state, see: https://github.com/pytorch/pytorch/issues/128427#issuecomment-2161496162. NOTE: Warnings issued under this context will still be cached in the __warningregistry__ and count towards the once/default rule. So you should NEVER use this on a user-land function. Filter must implement the showwarning API: def filter_fn(message, category, filename, lineno, file=None, line=None) -> bool: return True # show this warning entry c(|i|r |i|yyr'r^)rr filter_fnpriors r, _showwarningz#hide_warnings.._showwarning>s" d %f % 4 "6 " &r/N)warnings showwarning)rrrs` @r, hide_warningsr,s9  E#%+ $usA7 A AAcbeZdZdeeefdej dejddfdZdefdZ y) LazyStringfuncrrr4Nc.||_||_||_yr'rrr)r*rrrs r,rGzLazyString.__init__Js   r/cN|j|ji|jSr'rrNs r,__str__zLazyString.__str__Qs tyy$))3t{{33r/) rXrYrZrrr7rrrGrr^r/r,rrIsDRW%.0ggAC 44r/rstructured_logging_overhead time_spentcd}tjjjx}3|jj }|jj }|d|}|t|}t|xx|z cc<yyNr) rrorprq compile_idframe_idframe_compile_idr7r)r keyrr$r%s r,add_structured_logging_overheadr'Zs CMM00AACCP&&//#..?? !,-. #h#C(J6(r/cd}tjjjx}3|jj }|jj }|d|}|tj|Syr") rrorprqr#r$r%rr)r&rr$r%s r,get_structured_logging_overheadr)lss CMM00AACCP&&//#..?? !,-. *..s33r/cyr'r^r^r/r,rr{rr/r+rrr#c.tdfd||y)NrcdS)Nrr^)rr+sr,rz+trace_structured_artifact..s  r/)rrr#)r)r+rrr#s`` r,trace_structured_artifactr-xs  r/cyr'r^r^r/r,rrrr/)rrrrecord_logging_overheadr#rrrr/czgd}||vrtd|dt|stdt|t|stdt|tjrOt j }i} || |<|s0tjr+tjrtj| d<tjjj} |rS| Q|Otjj j#t%j&d j)| d <nt| r | j*n|} | Q| j,| j,| d <| j.| j.| d <| j0| j0| d <| r| j2| d<|} | t5| t6sJt5| t8rddj;d| Dzdz} nd} t=j>| | d} tAjBd}|jE| jGd|jI| d<tjKd| | ddtM|| |r&t j |z dz }tO|yyy)z metadata is an arbitrary JSON compatible struct, but it's expected to not be too long (e.g., less than 1MB) payload is an arbitrary string, which can be arbitrarily long (but expected to have newlines so no lines are too long) ) rankcompiled_autograd_idr$r%attemptseverity timestamprtthreadzname z is reserved and cannot be usedz(metadata_fn should be callable, but got z'payload_fn should be callable, but got r1Nr)skipstackr2r$r%r3z[ z, c3FK|]}tj|ywr')rzr{)rYis r,r[z#trace_structured..s0P1A0Ps!z ]cht|tr t|Stdt |d)NzObject of type z is not JSON serializable)r6rIr\ TypeErrortype)objs r, json_defaultz&trace_structured..json_defaults4%c3/#'9,'-d3i[8QRr/r)rWindentF)usedforsecurityzutf-8 has_payloadr)r|r}r)extra stacklevelgeA)(rcallabler=rrtimetime_nsrjrkrlrmrrorprq_logging structuredfrom_tracebackrextractsummaryr#r2r$r%r3r6r7r\rrzr{hashlibmd5updateencode hexdigestdebugrr')r+rrrrr/r#reserved_names start_timer~rcidr}r?hstructured_logging_overhead_ss r,rrs~& N ~uTH,KLMM K 6tK7H6I J   J 5d:6F5G H  \\^ $&"}t   "t':':'1   #40 "-1\\^j-HC,O ) +,I J #qr/cyr'r^r^r/r,rrrr/rrrr/c4trt||||||yy)zu For logging more detailed information used for debugging. This may result in the program becoming slow. rYN)rr)r+rrrrr/s r,dtrace_structuredr[s&  !-+$;  r/rW)FFN)Fr')NN) contextlib functoolsrMimportlib.utilrrJrzr;ros.pathr7r&rrrrFr collectionsrcollections.abcrr dataclassesrrtypingrr r typing_extensionsr weakrefr torch._logging.structuredr torch._guardsr rrtorch.utils._tracebackrrrrXrrrSDEFAULT_LOG_LEVELrrrrrrrr]rrrr`rrr DEFAULT_LOGGINGrvrr[r7rr\Loggerrr8rrr r rescapeVERBOSITY_REGEXrrrrrr lru_cacherrrr/rr?r=r@r<rBrrrrrrrrrrrrrrcacherr rcontextmanagerrrfloatrr'r)objectr-rr[torch.distributedr{rjr^r/r,rts    #.())' #<4t_g! G  o . OO "(2 268./6 )  VDVD VDr #$#$ #$L} J ll << LL||  "!$$ !" ",0#&+$)#("!&!&m t $J t  Dj  4Z Djt * t *4Z * !"#$ %&'()*+,-./0 123456789:;<=>?@ABCDEFGHIJKLMN OPQR $JST#sTz/ "T )UVWXYZ[\ $]^"_`abcd!efghijklmn oD QT'..)Q6PT " gnn  hh  -.  :M0NOPQ   ?E<~"$C$C$N 88v chsmd.B@5   ""c3h"C"".AAAX=**X=v &'(" 9   # ( K,\[%w,,[%|( ((ASWA :>' '  8%%8 4 41>? eK S6\D001 eKeKeK"eKD eK eKXCG  5A"!$(  "d38nuS#X>>?  S6\D001    "   4 r/