MjUddlZddlmcmZddlmZddlmZgZe e e d<ejjGddZy)N)Tensor)2_scripted_functional_optimizer_deprecation_warning__all__ceZdZ ddeededeeefdededededed ed efd Zd ed edzfdZ deedzfdZ y)_FunctionalAdamparamslrbetaseps weight_decayamsgradmaximizeforeachfused_allow_empty_param_listc ftdd|kstd|d|kstd|d|dcxkrdksntd|dd|d cxkrdksntd |d d|kstd ||||d|d |d |_||_||_||_| |_tjjttjtttjffi|_ t|dk(r | s td d|i|_y)N) stacklevelzInvalid learning rate: zInvalid epsilon value: rg?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: )r r beta1beta2r z%optimizer got an empty parameter listr)r ValueErrordefaultsr rrrtorchjitannotatedictrstrstatelen param_group) selfrr r r r r rrrrs f/var/www/html/ia-enhanced/venv/lib/python3.12/site-packages/torch/distributed/optim/functional_adam.py__init__z_FunctionalAdam.__init__sJ ;aHby6rd;< <cz6se<= =eAh$$B58*MN NeAh$$B58*MN Nl";L>JK K1X1X(        YY''U\\4U\\@Q;R-R(SUWX v;! $;DE E%f-paramgradNcfg}g}g}g}g}g}tj|} |"|j||j|||jvri|j|<|j|} tjd| d<tj |tj | d<tj |tj | d<|jr(tj |tj | d<|j|} |j| d|j| d|jr|j| d|j| dtj5tj|||||||j| |j|jd|jd |jd |jd |jd |j|jdd dddy#1swYyxYw)zo Similar to step, but operates on a single parameter and optionally a gradient tensor. Nrstep memory_formatexp_avg exp_avg_sqmax_exp_avg_sqrrr r r r has_complexrrrr r r rr grad_scale found_inf)r is_complexappendr tensor zeros_likepreserve_formatr no_gradFadamrrrr) r#r'r(params_with_gradgradsexp_avgs exp_avg_sqsmax_exp_avg_sqs state_stepsr1r s r$ step_paramz_FunctionalAdam.step_paramFs  $& &&u-    # #E * LL   " "DJJu JJu%E!LL-E&M$//U%:%: E) #("2"2U%:%:#E, ||*/*:*:)>)>+&' 5!i()5./ <<  " "5)9#: ;5=) ]]_  FF  'mmG,mmG,==&!]]>:MM%( jj%    s BH''H0 gradientscX|jd}g}g}g}g}g}g}d} t|t|k7r*tddt|dzdt|zt|jd|D]v\} } |  | t j | z} |j | |j | | |jvri|j| <|j| } t jd| d<t j| tj | d <t j| tj | d <|jr(t j| tj | d <|j| } |j | d |j | d |jr|j | d |j | dyt j5tj|||||||j| |j|j d |j d|j d|j d|j d|j"|j$dddddy#1swYyxYw)NrFzEthe gradients passed in does not equal to the size of the parameters!zParams length: z. zGradients length: rr*r+r-r.r/rrr r r r0)r"r!rziprr4r5r r6r7r8r r9r:r;rrrr) r#rCrr<r=r>r?r@rAr1r'gradientr s r$r*z_FunctionalAdam.steps!!(+ $&  v;#i. (W#CK=34&s9~&678  #4#3#3H#=yI 2OE8#u//66  ''. X& *(*DJJu% JJu-E$)LL$5E&M','7'7U-B-B(E)$+0*:*:U-B-B+E,'||272B2B!1F1F3./ 5)i 01""5#67<<#**51A+BC""5=1A 2D]]_  FF  'mmG,mmG,==&!]]>:MM%( jj%    s BJ  J)) gMbP?)g?g+?g:0yE>rFFFFF) __name__ __module__ __qualname__listrfloattupleboolr%rBr*r&r$rrs %1!(-+.V +. +.UE\" +.  +.  +.+.+.+.+."&+.Z::ftm:xGd6D=1Gr&r)rtorch.optim._functionaloptim _functionalr:r,torch.distributed.optim._deprecation_warningrrrJr__annotations__rscriptrrNr&r$rUsN ## cqqqr&