
    '}h6              !          d dl Z d dl mZ ddlmZmZmZmZmZmZm	Z	m
Z
mZmZ d dlmZmZ ddgZd dZ G d	 de      Zd
e
 de de	 de d	e_        	 	 	 	 	 d!dee   dee   dee   dee   dee   dee   dee   dededededededededef dZdee   dee   dee   dee   dee   dee   dededededededededefdZdee   dee   dee   dee   dee   dee   dededededededededefdZy)"    N)Tensor   )
	Optimizer_use_grad_for_differentiable
_get_value_default_to_fused_or_foreach_get_scalar_dtype_view_as_real_differentiable_doc_foreach_doc_maximize_doc_capturable_doc)ListOptionalASGDasgdc                 h    t        | t        j                        st        j                  | |      S | S )Ndevice)
isinstancetorchr   tensor)xr   s     O/var/www/html/test/engine/venv/lib/python3.12/site-packages/torch/optim/asgd.py
_to_tensorr      s&    a&||Af--H    c            	       l     e Zd Z	 	 	 	 	 	 	 	 	 d	dee   dededef fdZ fdZd Zed
d       Z	 xZ
S )r   foreachmaximizedifferentiable
capturablec                     d|k  st        d|       d|k  st        d|       t        ||||||||	|
	      }t        |   ||       y )Ng        zInvalid learning rate: zInvalid weight_decay value: )	lrlambdalphat0weight_decayr   r   r    r!   )
ValueErrordictsuper__init__)selfparamsr#   r$   r%   r&   r'   r   r   r    r!   defaults	__class__s               r   r+   zASGD.__init__   sp     by6rd;<<l";L>JKK%)!

 	*r   c                    t         |   |       | j                  D ]f  }|j                  dd        |j                  dd       |j                  dd       |j                  dd       |d   D ]  }| j                  j                  |g       }t        |      dk7  s/t        j                  |d         s;t        |d         }t        j                  |t               |j                  	      |d<   t        j                  |d
         s0t        j                  |d
   t               |j                  	      |d
<   t        j                  |d         rt        j                  |d   t               |j                  	      |d<    i y )Nr   r   Fr    r!   r-   r   step)dtyper   etamu)r*   __setstate__param_groups
setdefaultstategetlenr   	is_tensorfloatr   r	   r   )r,   r8   grouppp_statestep_valr/   s         r   r5   zASGD.__setstate__1   sF   U#&& 	pEY-Z/-u5\518_ 	p**..B/w<1$ ??76?;#(#9*/,,xGXGZcdckck*l ??75>:).genL]L_hihphp)q ??74=9(-WT]J[J]fgfnfn(o	p	pr   c                 D   d}|d   D ]  }	|	j                   |t        j                  |	      z  }|j                  |	       |	j                   j                  rt        d      |j                  |	j                          | j                  |	   }
t        |
      dk(  rt        j                  d|	j                  t                     |
d<   t        j                  |d   |	j                  t                     |
d	<   t        j                  d|	j                  t                     |
d
<   t        j                  |	t        j                        |
d<   |j                  |
d
          |j                  |
d          |j                  |
d	          |j                  |
d           |S )NFr-   z&ASGD does not support sparse gradientsr    )r   r2   r1   r#   r3   r4   )memory_formatax)gradr   
is_complexappend	is_sparseRuntimeErrorr8   r:   zerosr   r	   r   ones
zeros_likepreserve_format)r,   r=   params_with_gradgradsmusaxsetasstate_stepshas_complexr>   r8   s              r   _init_groupzASGD._init_groupD   sQ   x 	2Avv!u//22 ''*66##&'OPPQVV$

1u:?$)KK188K\K^$_E&M#(<<dAHHTeTg#hE%L"'**RHYH["\E$K"'"2"2)>)>#E$K 

5;'

5;'E%L)""5=1+	2, r   c                 b   | j                          d}|$t        j                         5   |       }ddd       | j                  D ][  }g }g }g }g }g }g }	| j	                  |||||||	      }
t        ||||||	|d   |d   |d   |d   |d   |d   |d   |d	   |d
   |
       ] |S # 1 sw Y   uxY w)zPerform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr$   r#   r&   r%   r'   r   r   r    r!   )
r$   r#   r&   r%   r'   r   r   r    r!   rT   ) _cuda_graph_capture_health_checkr   enable_gradr6   rU   r   )r,   closurelossr=   rN   rO   rP   rQ   rR   rS   rT   s              r   r1   z	ASGD.step^   s    	--/""$ !y! && 	E!ECCDK**52BE3PSUY[fgK Gn;;Gn">2i(z*$%56 .'!	: A! !s   B%%B.)	g{Gz?g-C6?g      ?g    .Ar   NFFFN)__name__
__module____qualname__r   boolr+   r5   rU   r   r1   __classcell__)r/   s   @r   r   r      sp     "&$ + $+ + + +>p&4 "+ "+r   ah  Implements Averaged Stochastic Gradient Descent.

    It has been proposed in `Acceleration of stochastic approximation by
    averaging`_.

    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        lambd (float, optional): decay term (default: 1e-4)
        alpha (float, optional): power for eta update (default: 0.75)
        t0 (float, optional): point at which to start averaging (default: 1e6)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        zx

    .. _Acceleration of stochastic approximation by averaging:
        https://dl.acm.org/citation.cfm?id=131098

    r-   rO   rQ   rP   rR   rS   r   r   r    r!   rT   r$   r#   r&   r%   r'   c                   |t        | |d      \  }}|r)t        j                  j                         rt	        d      |r%t        j                  j                         st
        }nt        } || |||||||||||||	|
       y)znFunctional API that performs asgd algorithm computation.

    See :class:`~torch.optim.ASGD` for details.
    NF)	use_fusedz6torch.jit.script not supported with foreach optimizers)	r$   r#   r&   r%   r'   r   r    r!   rT   )r   r   jitis_scriptingrI   _multi_tensor_asgd_single_tensor_asgd)r-   rO   rQ   rP   rR   rS   r   r   r    r!   rT   r$   r#   r&   r%   r'   _funcs                     r   r   r      s    2 1&.TYZ
7599))+STTuyy--/!"!%r   c       	   
      Z   t        |       D ]  \  }}||   }|s|n| }||   }||   }||   }||   }t        j                  j                         si|rg|j                  r$|j                  r|j                  r|j                  s7|j
                  r$|j
                  r|j
                  r|j
                  sJ d       t        j                  |      r?t        j                  |      }t        j                  |      }t        j                  |      }|dz  }|
dk7  r|j                  ||
      }|r,|j                  d||z  z
         |j                  ||d       n6t        |      }|j                  d||z  z
         |j                  ||        |s|j                         dk7  r0|j                  |j                  |      j                  |             n|j                  |       |r`|j                  |d||z  |z  z   |	z  z         |j                  dt        j                   ||z
  t        j"                  |            z         ;t        |      }t%        |d||z  |z  z   |	z  z        }|j                  |       t%        dt'        d||z
        z        }|j                  |        y )NzSIf capturable=True, params, mus, etas, and state_steps must be CUDA or XLA tensors.r   r   r%   value)	enumerater   _utilsis_compilingis_cudais_xlarF   view_as_realaddmul_addcmul_r   add_itemsubcopy_maximum	ones_liker   max)r-   rO   rQ   rP   rR   rS   r$   r#   r&   r%   r'   r   r    r!   rT   iparamrE   r4   rD   r3   step_t	eta_valuer1   new_etanew_mus                             r   rf   rf      sW   $ f% /5Qx#t$VV1gQ ||((*zMMbjjS[[V^^szzfmmede  E"%%d+D&&u-E##B'B 	!188E86DJJq53;'NN4BN/"3IJJq59,,-JJtI:J. aGGEIIbM&&r*+HHUOIIbQf!44>?@HHQv{EOOF4KLLMf%D EBJ,=(=%'G!HIGIIgC4"9$5 56FHHV_/r   c       	   
         t        |       dk(  ry |rJ d       t        j                  j                         s'|r%t	        d t        | |||      D              sJ d       t        j                  | |||||g      }|j                         D ]  \  \  }}\  \  }}}}}}}|rt        |||       |rt        j                  |      }|d   j                  r.t        j                  |t        j                  dd      d       nt        j                  |d	       |
dk7  rN|rt        j                  |||
       |}nt        j                  |||
      }t        j                  |||       nt        j                  |||      }t        j                  |||d
       ~t        j                   ||      }t        j                  |||       ~|rt        j                   ||      }t        j"                  |d       t        j$                  |       t        j&                  ||       ~t        j(                  ||	      }t        j*                  ||       t        j*                  ||       t        j                  |d	       t        j$                  |       t        j*                  ||       t        j&                  ||       5|d   j-                         }g }g }t/        t        |            D ]]  }t1        |d	||z  ||	z  z  z   z  |      }|j3                  |       t1        d	t5        d	||z
        z  |      }|j3                  |       _ t        j&                  ||       t        j&                  ||        y )Nr   z#_foreach ops don't support autogradc              3      K   | ]?  \  }}}}|j                   xr( |j                   xr |j                   xr |j                    A y wr[   )rq   ).0r>   r4   r3   r1   s        r   	<genexpr>z%_multi_tensor_asgd.<locals>.<genexpr>9  sD      P'q"c4 99LLLL Ps   AAzLIf capturable=True, params, mus, etas, and state_steps must be CUDA tensors.g      ?cpur   rj   r   rk   rl   )r:   r   ro   rp   allzipr   "_group_tensors_by_device_and_dtypeitemsr
   _foreach_negis_cpu_foreach_add_r   _foreach_add_foreach_addcmul__foreach_sub_foreach_maximum__foreach_reciprocal__foreach_copy__foreach_pow_foreach_mul_rx   ranger   rG   r}   )r-   rO   rQ   rP   rR   rS   r$   r#   r&   r%   r'   r   r    r!   rT   grouped_tensorsr   rg   grouped_paramsgrouped_gradsgrouped_axsgrouped_musgrouped_etasgrouped_state_stepsintermediatenew_musnew_etasr1   r~   r   r   s                                  r   re   re      sB   $ 6{aDDD <<$$&: P+.vsD++NP P 	[Z	[ P  BBFESVX[]acnCopO4C4I4I4KN7 	1&! 0 ,{K	*Q.-E!..}=M q!(( 3U\\#e5T\_` 3Q7 1##M>V,$11-WcdnEJ --m^SXYL 	lRTU )).+F\;G(()<bAG##GS1&&w/  g6 ))*=uEH%0"-!,&&x0"-  x8&q)..0DHG3{+, '$!ebj45=889& (#AAtby(9$9&Iv&'   x8  g6]N7r   r[   )NFFFF)r   r   	optimizerr   r   r   r   r	   r
   r   r   r   r   typingr   r   __all__r   r   __doc__r_   r<   r   rf   re   rB   r   r   <module>r      s    ) ) ) "6
y9 yx	 
 		 		 		 !B # 4L4<4 
f4 
f	4
 v,4 f4 d^4 4 4 4 4 4  	!4" 	#4$ %4& '4nALA<A 
fA 
f	A
 v,A fA A 	A 	A A A A A A  !AHl7Ll7<l7 
fl7 
f	l7
 v,l7 fl7 l7 	l7 	l7 l7 l7 l7 l7 l7  !l7r   