
    '}h8%                       d Z ddlmZ ddlZddlmZ ddlZddlmZm	Z	m
Z
 ddlmZ ddlmZmZmZ g dZ ej$                  ej&                  d	
      Z ed       e
j*                  d      ej,                  d d                     Z ed      ej,                  d!d d              Z ed      ej,                  d!d d              Z ed       e
j4                  d       e
j*                  dd      ej,                  d d                            Z ed       e
j*                  ddddddddd	      ej,                  	 	 d d                     Z ed      ej,                  d d              Z ed       e
j*                  ddddddd      ej,                  	 	 	 	 d"	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d#d                     Zej,                  	 	 	 	 	 	 d$d       Zej,                  	 	 	 	 	 	 	 	 d%d       Z y)&a&  This file exports ONNX ops for opset 14.

Note [ONNX operators that are added/updated in opset 14]
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
New operators:
    HardSwish, Trilu

Updated operators:
    Reshape
    Add, Sub, Mul, Div
    GRU, LSTM, RNN
    BatchNorm, Cumsum, Relu
    )annotationsN)Optional)
_constants_type_utilssymbolic_helper)GLOBALS)	_beartype	jit_utilsregistration)	hardswishtriltriureshape
batch_normquantized_hardswishscaled_dot_product_attention   )opsetzaten::hardswishvc                &    | j                  d|      S )N	HardSwishop)gselfs     Z/var/www/html/test/engine/venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset14.pyr   r   (   s     44T""    z
aten::trilc                ,    | j                  d||d      S )NTrilur   upper_ir   r   r   diagonalouts       r   r   r   /        44x433r   z
aten::triuc                ,    | j                  d||d      S )Nr      r    r   r"   s       r   r   r   5   r%   r   zaten::reshapeTc                4    t        j                  | ||d      S )Nr   )	allowzero)r   _reshape_helper)r   r   shapes      r   r   r   ;   s     **1dEQGGr   zaten::batch_normifc
                    t        j                         rFt        j                  |||||g      s,t        j
                  dk  rt        j                  dddd|      S t        j                  |d       t        j                  | |||||      \  }}}}| j                  d||||||d|z
  |sdnd|sdnd	
      }
|s|
S |
\  }}}|j                  |j                                |j                  |j                                |S )
N   BatchNormalizationr   zaAll input tensors must have the same `dtype`. Turn off Autocast or export using opset version 15.r   r'   r      )	epsilon_f
momentum_ftraining_mode_ioutputs)torchis_autocast_enabledr   args_have_same_dtyper   export_onnx_opset_version _onnx_opset_unsupported_detailedcheck_training_mode_batchnorm_helperr   setTypetype)r   inputweightbiasrunning_meanrunning_vartrainingmomentumepscudnn_enabledr$   resnew_running_meannew_running_vars                 r   r   r   E   s"     	!!#44FD,<
 --2?? C
 	
 '',?.=.O.O	5&$k/+FD, $$x<!)q!q  C 
14.  !2!2!45 0 0 23
r   zquantized::hardswishc                    t        j                  | |      \  }}}}t        | |      }t        j                  | |||      S N)r   dequantize_helperr   quantize_helper)r   xop_scaleop_zero_point_outputs         r   r   r   }   s@     !221a8JAq!Qq!_F**1fhNNr   z"aten::scaled_dot_product_attentionbc                   |r|rt        j                  |      sJ d       t        j                  |d      }t        j                  |      rt        | |      }|rt	        | ||      }t        j
                  |      }t        t        |            }	|	d   |	d   c|	d<   |	d<   | j                  d||	      }
| j                  d|| j                  d|            }| j                  d|
| j                  d|            }| j                  d	||      }t        j                  |      r|}net        j                  j                  |      t        j                  j                  k(  r| j                  d
t        j                  dg            }| j                  d
t        j                  t        d       g            }| j                  d|||      }| j                  d||      }nt        j                  j                  |      t        j                  j                   t        j                  j"                  t        j                  j$                  fv r| j                  d||      }n+t'        dt        j                  j                  |             | j                  d|d      }|dk7  rG| j                  d|| j                  d
t        j                  |t        j                                    }| j                  d	||      S )Nz6is_causal and attn_mask cannot be set at the same timer-   	Transpose)perm_iMulSqrtMatMulConstant        value_tinfWhereAddz Unsupported type for attn_mask: Softmaxaxis_ir   Dropoutdtype)r   _is_none_maybe_get_const_attention_scale_causal_attention_mask_get_tensor_ranklistranger   r   JitScalarType
from_valueBOOLr6   tensorfloatFLOATHALFBFLOAT16
ValueError)r   querykeyvalue	attn_mask	dropout_p	is_causalscalekey_shape_builtinkey_transposed_axeskey_transposedquery_scaledkey_transposed_scaledmul_qk
mul_qk_add
const_zeroconst_neg_infattn_weights                     r   r   r      s    o..y9@?@  ,,UC8E& E**1eS9	
 (88=u%678BB 504 TT+s3FTGN 44uadd65&9:LDDVU8KLTT(L*?@F	*
!!,,Y7$$))	* TT*ellC5.ATB
Ze}o1NODD)ZG	TT%3
		"	"	-	-i	8!!''!!&&!!**= 

 TT%3
.{/H/H/S/ST]/^._`
 	
 $$y*R$8KA~ddDDU\\)5;;%ODP
 44+u--r   c                   | j                  d|      }| j                  d|| j                  dt        j                  dgt        j                              | j                  dt        j                  t        j
                  gt        j                                    }| j                  d|t        j                  j                  |      j                               }| j                  dt        j                  d	gt        j                              }| j                  d
|| j                  d|            }| j                  d|t        j                  j                  |      j                               }|S )zCalculate the scale factor for the attention result.

    Args:
        query: Tensor of shape [..., L, E]

    Returns:
        Scalar scale factor := 1 / math.sqrt(query.size(-1))
    ShapeSlicer]   rW   rh   r_   Cast)to_i      ?Divr[   )r   r6   rt   int64r   	INT64_MAXr   rq   rr   	onnx_typeru   )r   rz   query_shapequery_shape_lastembedding_size	const_oner   s          r   rl   rl      s/    $$w&Ktt	Zrd%++!FG	j.B.B-C5;; W 	 	
	 TT&&11%8BBD  N
 Zse5;;)OPIDD	144#?@EDD&&11%8BBD  E
 Lr   c                :   | j                  d|      }| j                  d|      }| j                  dt        j                  dgt        j                              }| j                  dt        j                  dgt        j                              }| j                  d|||      }| j                  d|||      }| j                  d||d	
      }	| j                  dt        j                  dg            }
| j                  d|
|	      }| j                  d|d	      }| j                  dt        j                  dg            }| j                  dt        j                  t	        d       g            }| j                  d| j                  d||      ||      }|S )a  Create a causal mask for the given query and key tensors.

    Equivalent to::
        mask = torch.ones(L, S, dtype=torch.bool).tril(diagonal=0)
        attn_mask = torch.zeros(L, S, dtype=torch.float)
        attn_mask = attn_mask.masked_fill(not mask, -float('inf'))

    Args:
        query: Tensor of shape [..., L, E]
        key: Tensor of shape [..., S, E]

    Returns:
        Tensor of shape [L, S]
    r   r]   rW   rh   r_   rV   r   Concatr   re   r   Expandr   r    r^   ra   rb   Equal)r   r6   rt   r   ru   )r   rz   r{   r   	key_shapelast_idxsecond_last_idxtarget_lengthsource_lengthsizer   r}   r   r   s                 r   rm   rm      sW   & $$w&KWc"IttJbT(MtNHdd:u||RD/TdUODD+IMDD)_hGM44-q4ADZse)<=IXy$/IWi3Ij%,,u*=>JDDU\\E%L=/-JDKMgy*5}jI r   )r   jit_utils.GraphContextrL   )Nr^   FN)r   r   rz   torch._C.Valuer{   r   r|   r   r}   Optional[torch._C.Value]r~   ru   r   boolr   r   )r   r   rz   r   returnr   )r   r   rz   r   r{   r   r   r   )!__doc__
__future__r   	functoolstypingr   r6   
torch.onnxr   r   r   torch.onnx._globalsr   torch.onnx._internalr	   r
   r   __all__partialonnx_symbolic_onnx_symbolic
parse_argsbeartyper   r   r   quantized_argsr   r   r   r   rl   rm    r   r   <module>r      s    #    ? ? ' C C #""<#=#=RH !"C 
#  ! ## 
4  4 
4  4  %C%
H  & & !H "#Cc3S#sCH
22  I $2j &'
O  (O 45Cc3S#>
 +/&*E.E.E. 
E. 	E.
 (E. E. E. $E.  ? 6E.P !!&4!! !H %%&4%;I%% %r   