Mj9UddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl m Z ddl m Z ddlmZddlmZmZddlmZmZddlmZmZdd lmZmZej8eZej>e d <d Z!e"e d <ed GddZ#ed GddZ$ed GddZ%GddeZ&de'e(de"de'e#fdZ)de'e(de"de'e#fdZ* d6de(de(de"de+e'e(e'e#ffdZ,de(fd Z-d!e'e(d"e'e#de(dzfd#Z.d$Z/d%Z0d&Z1Gd'd(Z2Gd)d*eZ3Gd+d,eZ4Gd-d.Z5 d7d/e6d0e(dzd1e"d2e'e&d3e7e(d4e"ddfd5Z8y)8N)ABCabstractmethod)Callable)ThreadPoolExecutor) dataclass)BaseHTTPRequestHandlerThreadingHTTPServer)parse_qsurlparse) DictLoader Environment)get_world_sizetcpstore_clientloggerN@_DEFAULT_FETCH_TIMEOUTT)slotsc.eZdZUeed<eed<dZdZy)Response status_codetextcl|jdk7r%td|jd|jy)NzHTTP : )r RuntimeErrorrselfs `/var/www/html/ia-enhanced/venv/lib/python3.12/site-packages/torch/distributed/debug/_frontend.pyraise_for_statuszResponse.raise_for_status#s8   s "t'7'7&8499+FG G #c@tj|jSN)jsonloadsrrs rr#z Response.json'szz$))$$r N)__name__ __module__ __qualname__int__annotations__strrr#r rrrs IH%r rc"eZdZUeed<eed<y)NavLinkpathlabelN)r%r&r'r*r)r+r rr-r-+s I Jr r-c.eZdZUeed<edgefed<y)Router.HTTPRequestHandlerhandlerN)r%r&r'r*r)rbytesr+r rr1r11s I +,e3 44r r1ceZdZUeZeed<edee fdZ edee fdZ de eeffdZdedzfdZdefdZy) DebugHandler fetch_timeoutreturncyr"r+rs rrouteszDebugHandler.routes:s%(r cyr"r+rs r nav_linkszDebugHandler.nav_links=s*-r ciSr"r+rs r templateszDebugHandler.templates@s r Ncyr"r+rs rdumpzDebugHandler.dumpCsr cHt|jjSr")typer%lowerrs r dump_filenamezDebugHandler.dump_filenameFsDz""((**r )r%r&r'rr7floatr)rlistr1r:r-r<dictr*r>r@rDr+r rr6r67sj1M51(U ((-4=--4S>cDj+s+r r6urlstimeoutr8cddld}dtdtffd }t|5}t |j ||}ddd|S#1swYSxYw)Nrurlr8c j|}t|j|jS#jj $r}tdd|cYd}~Sd}~wjj $r}tdd|cYd}~Sd}~wt$r.}tdt|jd|cYd}~Sd}~wwxYw)NrIz Timeout: zConnectionError: r) postrrr exceptionsTimeoutConnectionError ExceptionrBr%)rLresperequestsrIs rgetzfetch_thread_pool..getUs ===g=6DD,,dii8 8""** 2C9QC1 1""22 :C#4QC!89 9 =CDG$4$4#5Rs!;< < =s?26C A#C #C ?B C  C #CC C ) max_workers)rYr*rrrFmap)rHrIr[rZexecutorrespsrYs ` @rfetch_thread_poolr_Os]K = = =  4.X\\#t,-. L. Ls AAcddldjdtdtffd dttdttffd }t j ||S)NrsessionrLr8c4K |j|4d{}|jd{}t|j|cdddd{S7C7-7 #1d{7swYyxYw#tj $r}tdd|cYd}~Sd}~wj $r.}tdt|jd|cYd}~Sd}~wt$r.}tdt|jd|cYd}~Sd}~wwxYww)NrOzTimeoutError: rPrrQ) rRrrstatusasyncio TimeoutError ClientErrorrBr%rV)rarLrWrrXaiohttps rfetchzfetch_aiohttp..fetchks =||C( 3 3D!YY[( T2 3 3 3( 3 3 3 3## 7C>!!56 6"" =CDG$4$4#5Rs!;< < =CDG$4$4#5Rs!;< < =sDA9AA9A$A A$ A9A"A9DA9 A$"A9$A6*A- +A62A95D6A99D B DD D2#CDD D'#D D DDDrHc 8Kj}j|4d{}ttj|Dcgc] }|| c}d{cdddd{S7Mcc}w7 7#1d{7swYyxYww)N)totalrN) ClientTimeout ClientSessionrFrdgather)rHclient_timeoutrarLrgrhrIs rrmzfetch_aiohttp..gatherws ..W.=(((@ V VGgnnd.SsuWc/B.STTU V V V.ST V V V Vsb)BA:BB A<B B !B( B4B5B<BBB B BB)rgrlr*rrFrdrun)rHrIrmrgrhs ` @@r fetch_aiohttprpfs[ =W22 = = =V49VhV ;;vd| $$r endpointargsc>t}ttDcgc]}d| }}|j|}|Dcgc]}|j d|d|}} t ||}||fScc}wcc}w#t $rt||}Y||fSwxYw)Nrankz /handler/?rN)rranger multi_getdecoderp ImportErrorr_) rqrrrIstorerkeysaddrsaddrr^s r fetch_allrs  E %n&6 7 81d1#J 8D 8 OOD !EFK Ld iz4& 9 LE L:eW5 %< 9 L :!%9 %<:s A6A;% BBBblobcZtj|}tj|dS)N)indent)r#r$dumps)rparseds r format_jsonrs! ZZ F ::fQ ''r r}r^c0t|Dcgc]\}}|jdk7s||f}}}|syt|}|t|z }d|d|dg}|D]&\}} |jd|d| j(dj |Scc}}w) zHReturn a summary string if any workers failed, or None if all succeeded.rNzPARTIAL DATA: /z workers respondedz Rank r ) enumeraterlenappendrjoin) r}r^ir{failedrjoklinesrtrWs rformat_fetch_summaryrs!*5!1 JAQ]]c5Iq!f JF J  JE V BbT5'); < =E4 d wtfBtyyk234 99U Ks BBac {% block title %}{% endblock %} - PyTorch Distributed
{% block header %}{% endblock %} {% block content %}{% endblock %}
a {% extends "base.html" %} {% block header %}

{% block title %}{{title}}{% endblock %}

{% endblock %} {% block content %} {% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ resp.text }}
{% endif %} {% endfor %} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}{{ title }}{% endblock %}

{% endblock %} {% block content %} {% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ format_json(resp.text) }}
{% endif %} {% endfor %} {% endblock %} c FeZdZ d deedededdfdZd dZd dZ d d Z y) PeriodicDumperhandlers output_dirinterval_secondsr8Ncn||_||_||_tj|_d|_yr") _handlers _output_dir_interval_seconds threadingEvent _stop_event_thread)rrrrs r__init__zPeriodicDumper.__init__s1 "%!1$??,04 r ctj|jdtj|j dd|_|j jy)NT)exist_okz distributed.debug.PeriodicDumper)targetdaemonname)osmakedirsrrThread_runrstartrs rrzPeriodicDumper.startsG D$$t4 ''993  r c|jj|j|jjyyr")rsetrrrs rstopzPeriodicDumper.stop&s3  << # LL    $r c|jjs|jD]} |j}|tjd}|jd|d}tjj|j|} t|d5}|j|ddd|jj!|j"|jjsyy#t$r(t j d|jYwxYw#1swYxYw#t$rt j d|Y9wxYw)NzFailed to dump %sz %Y%m%d_%H%M%S_z.txtwzFailed to write dump to %s)ris_setrr@rVr exceptionrDtimestrftimerr.rropenwritewaitr)rr3content timestampfilenamer.fs rrzPeriodicDumper._run+s9""))+>> I%llnG? MM/: %3356a {$Gww||D$4$4h?IdC)A() I    ! !$"8"8 9#""))+!$$%8':O:O:QR)) I$$%A4HIs;C> D> D22D>>-D/.D/2D; 7D>>E! E!rr8N) r%r&r'rFr6r*rErrrrr+r rrrsF #' 5|$ 5 5 5  5 :r rcReZdZUejZej ed<dZe ed<y)_IPv6HTTPServeraddress_familyirequest_queue_sizeN) r%r&r'socketAF_INET6r AddressFamilyr)rr(r+r rrrEs!+1??NF((:""r rc veZdZUded<dZdZdefdZdeee effdZ defdZ d efd ed e d e de fd Zy )r2FrontendServerfrontendcRtjd|jd||zy)Nz%s %sr)rinfoclient_address)rformatrrs r log_messagezHTTPRequestHandler.log_messageMs&     " TM r c:|jj|yr")r_handle_requestrs rdo_GETzHTTPRequestHandler.do_GETTs %%d+r r8c@t|jjSr")r r.rs rget_pathzHTTPRequestHandler.get_pathWs "'''r c4t|jSr")r get_raw_queryrs r get_queryzHTTPRequestHandler.get_queryZs**,--r c@t|jjSr")r r.queryrs rrz HTTPRequestHandler.get_raw_query]s "(((r NrdefaultrBcJ|j}||vr|S|||dS)Nr)r)rrrrBrs r get_query_argz HTTPRequestHandler.get_query_arg`s0  u NE$KN##r )r%r&r'r)rrr*rrGrFrrobjectrBrr+r rr2r2Jso ,(#(.4T#Y/.)s),0c$$"($7;$ $r r2c`eZdZ d dedeedzfdZddZddZde ddfd Z d e d e de fd Zy)rNportrc| ddlm}|}djd|D}ttt d}|D]!}|j |j#t|}t|d|_ |jjj ttt|i|_|D]:}|j!D]%}|j"|j|j$<'<||_t)d t*fd |i} d |f} t-| | |_t1j2|j4d dd |_|j6j9y)Nr)default_handlersrc3K|]6}|jD]!}d|jd|jd#8yw)z z N)r<r.r/).0r3links r z*FrontendServer.__init__..usN ))+ DII;b 4L M M s<>)z base.htmlz raw_resp.htmlzjson_resp.htmlT)loader enable_async)ziprrr<r2rr+z distributed.debug.FrontendServer)rrrrr)'torch.distributed.debug._debug_handlersrr BASE_TEMPLATERAW_RESP_TEMPLATEJSON_RESP_TEMPLATEupdater>r r _jinja_envglobalsrrr_routesr:r3r.rrBr2r_serverrr_serverr) rrrrnav_html all_templatesr3rrouteRequestHandlerClassserver_addresss rrzFrontendServer.__init__jsj   P')H99 #  '.0)   6G  !2!2!4 5 6M*%V$G &&# ' JL  9G ) 9+0== UZZ( 9 9"#  !    d&~7JK  '';;3   r r8c |jjy#t$rtj dYywxYw)Nz got exception in frontend server)r serve_foreverrVrrrs rrzFrontendServer._serves8 A LL & & ( A   ? @ As >>c8|jjyr")rrrs rrzFrontendServer.joins r reqc|j}||jvr|jdd|y|j|} ||}|jd|jdd|j|jj|y#ttf$r>}t j d||jddt|Yd}~yd}~wwxYw) NizHandler not found: z-Exception in frontend server when handling %siz Exception: rz Content-typez text/html) rr send_errorrV SystemExitrrrepr send_response send_header end_headerswfiler)rrr.r3rWrXs rrzFrontendServer._handle_requests||~ t|| # NN3"5dV < = ,,t$ 3rs   #$1!C+*J+**844 $$  % % %   555 +3+0DI$x..%S %E%d8n%4 "   /4  49d8n $%  (c( S $x. S4Z $> @$0-:-:j#)# $/$>_P_PP & &Dj&&< & s8 &  & &r