'}h;ddlZddlmZddlmZmZmZmZmZmZm Z m Z m Z m Z ddl mZmZddgZGddeZd d e d e d ed e d ze_ dd eedeedeedeedeedeedededededededededefdZd eedeedeedeedeedededededededededefdZd eedeedeedeedeedededededededededefdZy) N)Tensor) Optimizer_use_grad_for_differentiable _get_value_default_to_fused_or_foreach_get_scalar_dtype_differentiable_doc _maximize_doc _foreach_doc _view_as_real_capturable_doc)ListOptionalAdamaxadamaxc neZdZ d dddddeedededeffdZfdZd Zed d Z xZ S) rF)maximizedifferentiable capturableforeachrrrc Bd|kstd|d|kstd|d|dcxkrdksntd|dd|dcxkrdksntd|dd|kstd |t|||||||| } t | || y) NzInvalid learning rate: zInvalid epsilon value: r?z#Invalid beta parameter at index 0: rz#Invalid beta parameter at index 1: zInvalid weight_decay value: )lrbetaseps weight_decayrrrr) ValueErrordictsuper__init__) selfparamsrrrrrrrrdefaults __class__s Q/var/www/html/test/engine/venv/lib/python3.12/site-packages/torch/optim/adamax.pyr"zAdamax.__init__ sby6rd;< <cz6se<= =eAh$$B58*MN NeAh$$B58*MN Nl";L>JK K%)!   *c0t|||jD]}|jdd|jdd|jdd|jdd|dD]}|jj |g}t |dk7s.tj|drGt|d}|dr*tj|t|j ntj|t |d<y) NrrFrrr$rstepdtypedevicer,) r! __setstate__ param_groups setdefaultstategetlentorch is_tensorfloattensorr r-)r#r2grouppp_statestep_valr&s r'r/zAdamax.__setstate__1s U#&& _E   Y -   Z /   -u 5   \5 18_ _**..B/w<1$U__WV_-M$WV_5HmrsnAu||HDUDW`a`h`h'i,1LLIZI\,]FO  _ _r(cd}|dD]o}|j|tj|z}|j||jjr t d|j|j|j |} t| dk(r|dr*tjdt|jntjdt | d <tj|tj | d <tj|tj | d <|j| d |j| d |j| d r|S)NFr$z(Adamax does not support sparse gradientsrrr+rr.r*) memory_formatexp_avgexp_inf)gradr5 is_complexappend is_sparse RuntimeErrorr2r4zerosr r-r8 zeros_likepreserve_format) r#r9params_with_gradgradsexp_avgsexp_infs state_steps has_complexr:r2s r' _init_groupzAdamax._init_group?sL x .Avv~ 5++A. .K  # #A &vv"#MNN LL JJqME5zQ$),$7"'R7H7JSTS[S[!\=B\\#UfUh=if #(#3#3U%:%:$i $)#3#3U%:%:$i  OOE), - OOE), -   uV} -1 .4r(cz|jd}|$tj5|}ddd|jD]g}g}g}g}g}g}|d\} } |d} |d} |d} |d}|d}|d}|d }|j ||||||}t |||||| | | | | ||||| i|S#1swYxYw) zPerforms a single optimization step. Args: closure (Callable, optional): A closure that reevaluates the model and returns the loss. Nrrrrrrrr) rbeta1beta2rrrrrrrO) _cuda_graph_capture_health_checkr5 enable_gradr0rPr)r#closurelossr9rJrKrLrMrNrRrSrrrrrrrrOs r'r*z Adamax.step]s! --/  ""$ !y !&&" E! EHHK >LE5,CtB 0LI&GZ(H"#34N|,J**52BE8U]_jkK  )!-%' %" H O ! !s B11B:)gMb`?)g?g+?g:0yE>rNN) __name__ __module__ __qualname__rboolr"r/rPrr* __classcell__)r&s@r'rr su  "&"+$ "+$"+"+"+"+H _<"2"2r(aImplements Adamax algorithm (a variant of Adam based on infinity norm). .. math:: \begin{aligned} &\rule{110mm}{0.4pt} \\ &\textbf{input} : \gamma \text{ (lr)}, \beta_1, \beta_2 \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)}, \: \lambda \text{ (weight decay)}, \\ &\hspace{13mm} \epsilon \text{ (epsilon)} \\ &\textbf{initialize} : m_0 \leftarrow 0 \text{ ( first moment)}, u_0 \leftarrow 0 \text{ ( infinity norm)} \\[-1.ex] &\rule{110mm}{0.4pt} \\ &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ &\hspace{5mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ &\hspace{5mm}if \: \lambda \neq 0 \\ &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ &\hspace{5mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ &\hspace{5mm}u_t \leftarrow \mathrm{max}(\beta_2 u_{t-1}, |g_{t}|+\epsilon) \\ &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \frac{\gamma m_t}{(1-\beta^t_1) u_t} \\ &\rule{110mm}{0.4pt} \\[-1.ex] &\bf{return} \: \theta_t \\[-1.ex] &\rule{110mm}{0.4pt} \\[-1.ex] \end{aligned} For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_. a  Args: params (iterable): iterable of parameters to optimize or dicts defining parameter groups lr (float, optional): learning rate (default: 2e-3) betas (Tuple[float, float], optional): coefficients used for computing running averages of gradient and its square eps (float, optional): term added to the denominator to improve numerical stability (default: 1e-8) weight_decay (float, optional): weight decay (L2 penalty) (default: 0) z zd .. _Adam\: A Method for Stochastic Optimization: https://arxiv.org/abs/1412.6980 r$rKrLrMrNrrrrrOrrRrSrrc @td|Ds td|t||d\}}|r)tjj r td|r%tjj st }nt}||||||| | | | |||| |y)zrFunctional API that performs adamax algorithm computation. See :class:`~torch.optim.Adamax` for details. c3PK|]}t|tj ywrX) isinstancer5r).0ts r' zadamax..s@qz!U\\*@s$&zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF) use_fusedz6torch.jit.script not supported with foreach optimizers) rrRrSrrrrrOr)allrFrr5jit is_scripting_multi_tensor_adamax_single_tensor_adamax)r$rKrLrMrNrrrrrOrrRrSrr_funcs r'rrs2 @K@ @ ^  1&.TYZ 7599))+STTuyy--/#$   !%r(c t|D]B\}}||}| s|n| }||}||}||}tjjs9| r7|jr |js|j r |j sJd|dz }| dk7r|j || }tj|rTtj|}tj|}tj|}tj|}|j|d|z | sEtj|j||jj||ntj|j|jd|jj|j!dgd}|j#tj$|dd| r2||zdz }|j'|||z}|j)||d|t+|zz }||z }|j)||| Ey) NzGIf capturable=True, params and state_steps must be CUDA or XLA tensors.rralpha)outF)keepdim)value) enumerater5_utils is_compilingis_cudais_xlaaddrC view_as_reallerp_maximummul_absadd_cat unsqueeze unsqueeze_copy_amaxdiv_addcdiv_r)r$rKrLrMrNrrRrSrrrrrrOiparamrBr@rAstep_tnorm_bufneg_bias_correctiondenombias_correctionclrs r'riris"f%395Qx#t$1+1+Q||((*zMMfnn  YX Y !  1 88E86D   E "&&u-E%%d+D((1G((1G  dAI& MM U# $  yye$..q1488:??33G3R3RST3UVXYH MM%**Xq%@ A #(6/A"5   $ $R (11E NN7E *%:f+="==O&C NN7GC4N 8g39r(c F| rJdt|dk(rytjjs)| r't dt ||Ds t dtj|||||g}|jD] \\}}}}}}| rt||||| rtj|}|djr.tj|tjdddntj|d |dk7r3| rtj|||ntj|||}tj ||d |z tj"||| s|dk(rtj$|}ntj&|tj|| tj(||| rqtj*||}tj,|d tj.||tj0||}tj2||||Dcgc]}d |t5|zz }}|Dcgc] }||z d z }}tj2|||| ycc}wcc}w) Nz#_foreach ops don't support autogradrc3VK|]!\}}|jxr |j#ywrX)ru)rar:r*s r'rcz'_multi_tensor_adamax..\s$[wq$ 2dll2[s')z@If capturable=True, params and state_steps must be CUDA tensors.rcpu)r-rmr)r4r5rsrtreziprFr"_group_tensors_by_device_and_dtypevaluesr _foreach_negis_cpu _foreach_add_r8 _foreach_add_foreach_lerp_ _foreach_mul_ _foreach_abs _foreach_abs__foreach_maximum_ _foreach_pow _foreach_sub_ _foreach_div_ _foreach_mul_foreach_addcdiv_r)r$rKrLrMrNrRrSrrrrrrrOgrouped_tensorsgrouped_params grouped_gradsgrouped_exp_avgsgrouped_exp_infsgrouped_state_stepsrjbias_correctionsrr*bc step_sizes r'rhrhCsv$DDD  6{a LL % % 'J[#fkBZ[[]^^BBFES[]egrCstOixiijB3ce a.-)9;KM`cd  .-9IK[ \ !..}=M q ! ( (    3U\\#e5T\_ `    3Q 7 1 ##M>V % 2 2=.Xd e  -}a%iH ,e4LA-!..}=M    . M3/  0-@ $11%9LM     0! 4    0" 5&&'79IJE  # #N4De LJ]^$EZ-=$= =^ ^2BCB"r'RCIC  # #N4DFVXa bg3cb _Cs J,J)NFFFF)r5r optimizerrrrrr r r r r rtypingrr__all__r__doc__r\r7rrirhr>r(r'rs )))" X DYDN2       3+l# 8 L8 <86l86l 8 f 8d^88888 8 8 !8" #8$%8vD9 LD9 <D96lD96l D9 f D9 D9 D9 D9 D9D9D9D9D9D9NPc LPc <Pc6lPc6l Pc f Pc Pc Pc PcPc PcPcPcPcPcr(