
    |h                         d Z ddlmc mZ ddlmZ ddlmZ ddl	m
Z
  e        e
dg        G d d	ej                                      Zej                   j                  d
ej                        e_         y)zLion optimizer implementation.    N)	optimizer)register_keras_serializable)keras_exportzkeras.optimizers.Lion)v1c                   V     e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 d fd	Z fdZd Z fdZ xZS )Liona  Optimizer that implements the Lion algorithm.

    The Lion optimizer is a stochastic-gradient-descent method that uses the
    sign operator to control the magnitude of the update, unlike other adaptive
    optimizers such as Adam that rely on second-order moments. This make
    Lion more memory-efficient as it only keeps track of the momentum. According
    to the authors (see reference), its performance gain over Adam grows with
    the batch size. Because the update of Lion is produced through the sign
    operation, resulting in a larger norm, a suitable learning rate for Lion is
    typically 3-10x smaller than that for AdamW. The weight decay for Lion
    should be in turn 3-10x larger than that for AdamW to maintain a
    similar strength (lr * wd).

    Args:
        learning_rate: A `tf.Tensor`, floating point value, a schedule that is a
            `tf.keras.optimizers.schedules.LearningRateSchedule`, or a callable
            that takes no arguments and returns the actual value to use. The
            learning rate. Defaults to 0.0001.
        beta_1: A float value or a constant float tensor, or a callable
            that takes no arguments and returns the actual value to use. The
            rate to combine the current gradient and the 1st moment estimate.
        beta_2: A float value or a constant float tensor, or a callable
            that takes no arguments and returns the actual value to use. The
            exponential decay rate for the 1st moment estimate.
        {{base_optimizer_keyword_args}}

    References:
        - [Chen et al., 2023](http://arxiv.org/abs/2302.06675)
        - [Authors' implementation](
            http://github.com/google/automl/tree/master/lion)

    c                     t        |   d|||||||	|
|d	| | j                  |      | _        || _        || _        |dk  s|dkD  rt        d| d      y )N)	nameweight_decayclipnorm	clipvalueglobal_clipnormuse_emaema_momentumema_overwrite_frequencyjit_compiler      z	`beta_1`=zI must be between ]0, 1]. Otherwise, the optimizer degenerates to SignSGD. )super__init___build_learning_rate_learning_ratebeta_1beta_2
ValueError)selflearning_rater   r   r   r   r   r   r   r   r   r   r
   kwargs	__class__s                 [/var/www/html/test/engine/venv/lib/python3.12/site-packages/tf_keras/src/optimizers/lion.pyr   zLion.__init__>   s      	 	
%+%$;#	
 	
 #77FQ;&1*F8 $8 8  %    c                     t         |   |       t        | d      r| j                  ryg | _        |D ].  }| j                  j                  | j                  |d             0 d| _        y)zInitialize optimizer variables.

        Lion optimizer has one variable `momentums`.

        Args:
            var_list: list of model variables to build Lion variables on.
        _builtNm)model_variablevariable_nameT)r   buildhasattrr#   	momentumsappendadd_variable_from_reference)r   var_listvarr   s      r    r'   z
Lion.buildc   sm     	h4"t{{ 	CNN!!00#&c 1 	 r!   c           	         t        j                  | j                  |j                        }t        j                  | j                  |j                        }t        j                  | j
                  |j                        }| j                  |      }| j                  | j                  |      }t        |t         j                        r|j                  ||z         |j                  t        j                  |j                  d|z
  z  |j                               |j                  |t         j                   j#                  |      z         |j                  ||z  |z         |j                  t        j                  |j                  d||z  z
  z  |j                               y|j                  |t         j                   j#                  ||z  |d|z
  z  z         z         |j                  ||z  |d|z
  z  z          y)z=Update step given gradient and the associated model variable.g      ?N)tfcastr   dtyper   r   _var_keyr)   _index_dict
isinstanceIndexedSlicesassignscatter_addvaluesindices
assign_submathsign)r   gradientvariablelrr   r   var_keyr$   s           r    update_stepzLion.update_stepw   s   WWT''8hnn5hnn5--)NN4++G45h 0 01HHQZ MM  OOsV|4h6F6F
 RWW\\!_ 45HHQZ&()MM  OOsVf_'<=x?O?O RWW\\!f*x3</H"HII HHQZ(cFl";;<r!   c                     t         |          }|j                  | j                  | j                        | j
                  | j                  d       |S )N)r   r   r   )r   
get_configupdate_serialize_hyperparameterr   r   r   )r   configr   s     r    rC   zLion.get_config   sO    #%!%!?!?''" ++++	
 r!   )g-C6?g?Gz?NNNNFrG   NTr   )	__name__
__module____qualname____doc__r   r'   rA   rC   __classcell__)r   s   @r    r   r      sI    F  $#J(=> r!   r   z{{base_optimizer_keyword_args}})rK   tensorflow.compat.v2compatv2r/   tf_keras.src.optimizersr   'tf_keras.src.saving.object_registrationr    tensorflow.python.util.tf_exportr   	Optimizerr   replacebase_optimizer_keyword_argsr   r!   r    <module>rV      sv    % ! ! - O : %"-F9 F . FR ||##%y'L'Lr!   