
    '}hwg             !       g   U d dl Z d dlZd dlZd dlZd dlmZ d dl mZmZ d dlm	Z	m
Z
 d dlmZmZmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d dlmc mZ d dlmZm Z m!Z! d dl"m#Z# d dl$m%Z% d d	lm&Z&m'Z'm(Z(m)Z) d d
l*m+Z+m,Z,m-Z-m.Z. d dl/m0Z1 d dl2m3Z3 ejh                  jj                  Z5g Z6ee7   e8d<   ejr                  jt                  jv                  Z; G d de      Z<	 ddedejz                  de>fdZ? ee?ejz                  j                  d      ZA ee?ejz                  j                        ZB ee?ejz                  j                        ZDde!deEde!fdZF e#e;j                         e.d      eBde!de!fd                     ZG e#e;j                         e.d      eBde!de!fd                      ZH e#e;j                         e.d      eBde!de!d!eJd"eJfd#                     ZI e#e;j                         e.d      eBd$e!d%eJd&eJd'eJd(e>d)e!fd*                     ZK e#e;j                  j                  g      d+        ZN e#e;j                  jB                  g      d,e!fd-       ZO e#e;j                         e.       eBd.e!de!fd/                     ZP e#e;j                         e.d      eBd$e!d.e!fd0                     ZQ e#e;j                         e.d      d$e!d.e!d1eJd2eJfd3              ZR e#e;j                         e.       eBd.e!de!fd4                     ZS e#e;j                         e.       eBd$e!d.e!de!fd5                     ZT e#e;j                         e.d      d$e!d.e!d"eJfd6              ZU e#e;j                         e.d      eBd$e!d.e!d7eJd8e>fd9                     ZV e#e;j                         e.d      eBdd:e!d.e!d;e7fd<                     ZW e#e;j                        eBd$e!d=e!fd>              ZX e#e;j                         e.       eBd.e!de!fd?                     ZY e#e;j                         e.d      eBd$e!d.e!de!fd@                     ZZ e#e;j                        d.e!dAe!de!fdB       Z[ e#e;j                        d$e!d.e!dAe!dee!e!f   fdC       Z\ e#e;j                        e;j                  j                  j                  e5j                         e.       eB	 	 	 	 dd.e!dDe!dEeJdFeJdGe>dHeej                     de!fdI                            Z] e#e;j                        e;j                  j                  j                  e5j                        eB	 	 dd.e!dDe!dEeJdFeJdGe>dHeej                     de!fdJ                     Zb e#e;j                         e.       eBd$e!d.e!dDe!dEeJdFeJdGe>d8e>de!fdK                     Zc e#e;j                         e.d      eBd$e!d.e!dLe!de!fdM                     ZddNe!dOeEfdPZedQej                  fdRZg e#e;j                         e.       eBe<j                  j                  fd.e!dSe!dOeEde!fdT                     Zh e#e;j                         e.d      eBd$e!d=e!dSe!dOeEfdU                     Zk e#e;j                         e.       eBe<j                  j                  dVfd.e!dSe!dOeEd!eJfdW                     Zl e#e;j                  j                        eBd$e!d.e!dSe!dOeEd!eJf
dX              Zm e#e;j                  j                        eBd$e!d.e!dSe!dOeEd!eJde!fdY              Zo e#e;j                  j                        eBd$e!d.e!dSe!dOeEdZeJf
d[              Zp e#e;j                  j                        eBd$e!d.e!dSe!dOeEdZeJde!fd\              Zrd$e!d.e!dSe!dAee!   dOeEd]eEd^e!de!fd_Zs e#e;j                         e.d      eBd$e!d.e!deEde!fd`                     Zt e#e;j                         e.d      d$e!d.e!dSe!dAee!   dOeEd]eEd^e!de!fda              Zu e#e;j                         e.d      d$e!d.e!dSe!dAee!   dOeEd]eEd^e!de!fdb              Zv e#e;j                         e.       eBde<j                  j                  fd.e!dSe!dAee!   dOeEde!f
dc                     Zw e#e;j                         e.d      eBde<j                  j                  fd$e!d.e!dSe!dAee!   dOeEde!fdd                     Zx e#e;j                         e.       eBe<j                  j                  fd=e!dSe!dOeEde!fde                     Zy e#e;j                         e.d      eBe<j                  j                  fd$e!d.e!dSe!dOeEde!f
df                     Zz e#e;j                         e.       dd=e!dge!dheJfdi              Z{ e#e;j                         e.       dje!dke!de!fdl              Z| e#e;j                         e.       d$e!dmeeE   deEdneEdoeEdpeEfdq              Z} e#e;j                  jB                        	 	 	 	 dd.e!deEdneeE   doeeE   dpeEf
ds       Z e#e;j                          e.       d$e!dmeeE   deEdteEfdu              Z e#e;j                         e.       d$e!dmeeE   dveEdweEdxeEf
dy              Zd$e!de!dzej                  fd{Z e#e;j                         e.d      eAd$e!d|e!deEdzej                  fd}                     Z e#e;j                         e.       eAd$e!d|e!deEdzej                  fd~                     Zd Z e#e;j                         e.       d=e!deeE   deeE   deeE   deeE   de!fd              Z e#e;j                         e.       eBd=e!deeE   deeE   deeE   deeE   deeE   de!fd                     Z e#e;j                         e.       d$e!de!d&eJfd              Z e#e;j                         e.       d:e!deeE   deEdeEdpeEde!fd              Z e#e;j                  j                        eB	 dd$e!d.e!deeJ   de!fd              Z e#e;j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d=e!dheJdee>   fd                     Z e#e;j                         e.dd      d=e!dheJdee>   fd              Z e#e;j                         e.       de!deEde>fd              Z e#e;j                          e.       de!deEde>fd              Z e#e;j"                         e.       	 	 	 ddAe!de!deEde>de>de!fd              Z e#e;j$                         e.       d$e!de!deEdeEde>f
d              ZdeeE   fdZdee!   deEdeEdee!   fdZdee!   fdZdee!   deEfdZdee!   deEdeEfdZ e#e;j0                  j                  e;j0                  j                  g      	 ddee!   deEdeEdee!   de!f
d       Z e#e;j2                        	 dd.e!deeE   deEdee!   fd       Z e#e;j4                  j                  e;j4                  j                  g      	 	 dd.e!deeE   deEdeee!      deee!      f
d       Z e#e;j6                  jB                        dd=e!deEdeEdee!df   fd       Z e#e;j8                  j                        	 dd=e!deeE   deEdee!df   fd       Z e#e;j:                  jB                        dd.e!deEdeEdee!df   fd       Ze;j<                  j>                  j                  e5j                        	 dd.e!de!deEdee!df   fd       Z e#e;jB                         e.       eBdd.e!de!de!d!eEd%eEf
d                     Z e#e;jD                         e.       eB	 	 	 dd.e!de!de!d!eEd%eEde>fd                     Z e#e;jF                         e.       eBdd.e!de!de!d!eEd%eEf
d                     Z e#e;jH                  j                        eBd$e!d=e!de!de!dee!   deEdeEdeEdeEdee>   deee!   ee!   ee!   f   fd              Z e#e;jH                  j                        d$e!d=e!de!de!dee!   deEdeEdeEdeEdee>   dejB                  dejB                  dejB                  deee!   ee!   ee!   f   fd       Zdee!   dee!   fdÄZ e#e;jN                  j                        de!d=e!deeE   de!de!dAee!   dee!   dee>   deee!   ee!   ee!   f   fdǄ       Z e#e;jN                  j                        de!d=e!deeE   de!de!dAee!   dee!   dee>   dejB                  dejB                  dejB                  deee!   ee!   ee!   f   fdȄ       Zd=e!dAee!   dee!   dee!   dee!   dGe>deJdeJde>dee!e!e!ee!   ee!   f   fd̈́Z e#e;jT                         e.dddϫ      d=e!dAee!   dee!   dee!   dee!   dGe>deJdeJdee!e!e!f   fdЄ              Ze;jT                  j                  j                  e5j                        e;jT                  j                  j                  e5j                        d=e!dAee!   dee!   dee!   dee!   dGe>deJdeJdee!e!e!f   fdф              Ze;jX                  j                  j                  e5j                        ddee!   fd҄       Z e#e;j\                  j                        d=e!dAee!   dee!   de!de!deJdeJdee!e!e!f   fdӄ       Z e#e;j^                  j                        d=e!dAee!   dee!   de!de!dGe>deJdeJdee!e!e!f   fdԄ       Z e#e;j^                  j`                        d=e!dAee!   dee!   dGe>deJdeJdee!e!e!f   fdՄ       Z e#e;jd                  j                        d=e!dAee!   dee!   de!de!dGe>deJdeJdee!e!e!e!e!f   fdք       Z e#e;jf                         e.dd      eBddׄ                     Zd؄ Z e#e;jl                         e.       dddddddٜde!dQeej                     deejn                     de>de>deejp                     fdބ              Z e#e;jr                  e;jt                  e;jv                  g       e.       d߄               Ze;jz                  j                  j                  e5j                         e#e;jz                         e.dddd      d=e!dAe!dee!   dee!   dee!   dGe>deJdeJfd                     Zd Z e#e;j~                  j                        de!d=e!dAee!   dee!   dee!   dee!   dee!   de>deJdee>   dee!ee!   ee!   f   fd       Z e#e;j~                  j                        de!d=e!dAee!   dee!   dee!   dee!   dee!   de>deJdee>   dejB                  dejB                  dejB                  dee!ee!   ee!   f   fd       Z e#e;j                         e.ddd      d=e!d$e!dAe!dee!   dee!   dee!   dee!   deJde!fd              Z e#e;j                         e.       eBd=e!deeEeEf   fd                     Z e#e;j                        drdde(deEdte(de(d%e'f
d       Z e#e;j                         e.       drdde(deEdte(de(d%e'f
d              Zdrdde(deEdte(de(de>d%e'fdZ e#e;j                  j                        e;j                  j                  j                  e5j                        dd              Z e#e;j                        de(deEdte(de(fd       Z e#e;j                         e.       de(deEdte(de(fd              Zde(deEdte(de(de>f
dZ e#e;j                         e.d|dL      eBd.e!dee!e!f   fd                     Z e#e;j                         e.       	 	 	 dde!dee>eEeJf   dee>eEeJf   dHeej                     fd              Z e#e;j                        dd       Zd Zd Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d                      Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d                      Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d                      Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d                      Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d                       Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d                      ZܐddZ e#e;j                  j                        e;j                  j                  j                  e5j                        eB	 dd=e!deeE   deeJ   de!fd                     Z e#e;j                  j                        e;j                  j                  j                  e5j                        eB	 dd=e!deeE   deeJ   de!fd                     ZӐd Z e#e;j                  j                        e;j                  j                  j                  e5j                        eB	 	 dd=e!deeE   deeJ   deeJ   de!f
d	                     Z e#e;j                  j                        e;j                  j                  j                  e5j                        eB	 	 dd=e!deeE   deeJ   deeJ   de!f
d
                     Z e#e;j                  j                        e;j                  j                  j                  e5j                        eB	 	 	 dd=e!deeE   deeJ   deeJ   deeJ   de!fd                     Z e#e;j                  j                        e;j                  j                  j                  e5j                        eB	 	 	 dd=e!deeE   deeJ   deeJ   deeJ   de!fd                     Zېd Zߐd Zd Zd Z	 ddZd Zd ZddZddZd Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d                      Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d                      Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d                      Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d                      Zd ZddZddZd Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d                       Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d!                      Zd" Zd# Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d$                      Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d%                      Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d&                      Z e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d'                      Z  e#e;j                  j                         e#e;j                  j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        e;j                  j                  j                  e5j                        d(                                                         Z e#e;j                  j                  e;j                  j                  g       e.       	 dd=e!deeE   d)e>deeJ   de!f
d*              Z e#e;j                  j                  e;j                  j                  g      e;j                  j                  j                  e5j                         e.       	 	 dd=e!deeE   d)e>deeJ   deeJ   de!fd+                     Z e#e;j                  j                  e;j                  j                  g       e.       	 	 	 dd=e!deeE   d)e>deeJ   deeJ   deeJ   de!fd,              Zdd-Zd. ZeBd=e!deeE   d)e>deeeJ      de!f
d/       Z e#e;j                  j                        d0e!d1e!de>fd2       Z e#e;j                  e;j                  g       e.       d3               Z	 e#e;j                  g      d4        Zd.e!dSe!dAee!   dOeEd]eEdee!e!f   fd5Z e#e;j                         e.d|d^      d.e!dSe!dAee!   dOeEd]eEdee!e!f   fd6              Z e#e;j                         e.d|d^      d.e!dSe!dAee!   dOeEd]eEdee!e!f   fd7              Zde!d8eJde!fd9Zde!d8eJde!fd:Zd;e!de)fd<Zd=e)d>e!de!fd?Zd>ee!   de!fd@ZdAeEd)e>dQej                  dejn                  fdBZdCe!dDeEdEeEd)e>fdFZdCe!dGeEdDeEdEeEd)e>f
dHZdCe!deeE   d)e>fdIZdCe!deeE   d)e>fdJZ e#e;j4                         e.       eBdCe!deeE   d)e>fdK                     Z	 	 	 	 dd0e!dLe!dMeEdNeEd)e>dOe>de!fdPZ e#e;j8                         e.       eB	 	 	 dd0e!dLe!dMeEdNeEd)e>de!fdQ                     Z e#e;j:                         e.       eBdR                      Z e#e;j<                         e.       dde<j                  j                  fdS              ZdTejB                  dUejB                  dVe>de>fdWZe;j@                  j                  j                  e5j                         e.dX      ddYdZ              Z  e#e;jB                  j                        eB	 	 dd0e!deeEeEf   d)e>d[eeJ   d\eeJ   de!fd]              Z" e#e;jB                  j                        e;jB                  j                  j                  e5j                        e;jB                  j                  j                  e5j                         e.       eB	 dd0e!deeeEeEf      d)e>d^eeeJeJf      de!f
d_                                   Z# e#e;jH                         e#e;jJ                         e#e;jL                        eB e.       d0e!deeEdf   de!fd`                                   Z' e#e;jP                         e#e;jR                         e#e;jT                        eB e.       d0e!deeEdf   de!fda                                   Z+d0e!deeEdf   dbeeEeEeEge!f   de!fdcZ, e#e;jZ                         e.ddde      dddfdg              Z- e#e;j\                         e.       dddhdi              Z. e#e;j^                  j                  e;j^                  j                  g       e.       dej`                  dddjdoe'dQeej                     dkejb                  deejn                     de>f
dl              Z2 e#e;j^                  jf                  g      dej`                  dddjdne'doe'dQeej                     dkejb                  deejn                     de>fdm       Z4 e#e%      dn        Z5 e#e;jl                        e;jl                  j                  j                  e5j                         e.       drdrde<j                  j                  fd=e!dSe!dhe'doe'dAee!   dOeEde!fdp                     Z6 e#e;jn                        e;jn                  j                  j                  e5j                         e.d|dq      d=e!dSe!dOeEdee!e!f   fdr                     Z7 e#e;jp                  j                        	 	 ddddsdte!due!d,e!dveJdwe>dxee!   d&eeJ   dee!e!f   fdy       Z9dz Z: e#e;jv                  g       e.       eBdd{                     Z; e#e;jx                         e.       d|               Z< e#e;jz                        d}        Z= e#e;j|                  j                  e;j|                  j                  g      ddd~d.e!dQeej                     dee!   de!fd       Z? e#e;j                  j                  e;j                  j                  g      dd.e!deeE   fd       ZB e#ejt                  jv                  j                        dd       ZC e#e;j                         e.       dddd              ZDdddZEddddZF e#e;j                         e.       d               ZG e:e;j                  e;j                          e:e;j                  e;jB                          e:e;j                  e;jF                          e:e;j                  e;jv                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                          e:e;j                  e;j                         y(      N)Enum)partialreduce)chainproduct)AnyCallablecastIterableListOptionalTupleUnion)	sym_floatsym_intTensorregister_decomposition)	out_dtype)IntLike
NumberType
TensorLikeTensorSequenceType)_maybe_convert_to_dtype_maybe_resize_out_safe_copy_outout_wrapper)_pytree)tree_map__all__c                       e Zd ZdZdZdZy)	Reductionr         N)__name__
__module____qualname__NONEMEANSUM     [/var/www/html/test/engine/venv/lib/python3.12/site-packages/torch/_decomp/decompositions.pyr"   r"   $   s    DD
Cr,   r"   Fftype_promotioncompute_dtype_onlyc                 J     t        j                          fd       }|S )Nc                     t        j                  | i |D cg c]  }t        |t              s| }}t	        j
                  |di\  fd}fd} 
t        ||       i t        ||      }	r|S t        ||      S c c}w )Ntype_promotion_kindc                 J    t        | t              r| j                        S | S N
isinstancer   to)xcomputation_dtypes    r-   increase_precz0type_casts.<locals>.inner.<locals>.increase_prec<   s"    !V$tt-..r,   c                 J    t        | t              r| j                        S | S r5   r6   )r9   result_dtypes    r-   decrease_precz0type_casts.<locals>.inner.<locals>.decrease_precB   s!    !V$ttL))r,   )pytreearg_tree_leavesr7   r   utilselementwise_dtypesr   )argskwargsr9   	flat_argsr;   r>   rr:   r=   r0   r.   r/   s          @@r-   innerztype_casts.<locals>.inner2   s     --t>v>
*QPVBWA
	 
 +0*B*B+
,:+
'<
		 xt,P0OPHM1--1
s
   BB)	functoolswraps)r.   r/   r0   rG   s   ``` r-   
type_castsrJ   -   s'    
 __Q. .6 Lr,   T)r/   r0   )r/   r9   dimreturnc                 j    t        || j                         z
        D ]  }| j                  d      }  | S N)rangerK   	unsqueeze)r9   rK   _s      r-   _unsqueeze_to_dimrS   _   s2    3=! KKOHr,   
grad_inputout_gradyc                 4    | d||z  z
  j                         z  S Nr#   conj_physicalrU   rV   s     r-   tanh_backwardr\   e   s      q1q5y//111r,   c                 4    | |d|z
  z  j                         z  S rX   rY   r[   s     r-   sigmoid_backwardr^   l   s      qAE{11333r,   beta	thresholdc                 t    ||z  j                         }t        j                  ||z  |kD  | | |z  |dz   z        S N      ?)exptorchwhere)rU   r9   r_   r`   zs        r-   softplus_backwardrh   s   s=     
TA;;DI-xAS9QRRr,   grad_outputalphascaleinput_scale	is_resultself_or_resultc                     ||z  }|}|}|r&t        j                  |dk  | |z  ||z   z  | |z        S t        j                  |dk  | |z  |z  t        j                  ||z        z  | |z        S Nr   )re   rf   rd   )	ri   rj   rk   rl   rm   rn   negcoefposcoef
negiptcoefs	            r-   elu_backwardrt   {   s     emGGJ{{a*$(@A'!
 	
 {{a*$w.>J;V1WW'!
 	
r,   c                 .    t        j                  | |      S r5   )re   	full_likeselfvalues     r-   fill_scalarrz      s    ??4''r,   ry   c                     t        j                  j                         dk(  fd       t        j	                  |       S )Nr   c                  ,    d j                          dS )Nz@fill only supports 0-dimension value tensor but got tensor with z dimensionsrK   )ry   s   r-   <lambda>zfill_tensor.<locals>.<lambda>   s    RSXS\S\S^R__jk r,   )re   _checkrK   atencopyrw   s    `r-   fill_tensorr      s3    	LL		qk 99T5!!r,   rx   c                 f    t        j                  t        j                  | dz   d      d      dz  S N   r   min   maxre   clamprx   s    r-   hardsigmoidr      s)     ;;u{{4!83;a??r,   c                 H    t        j                  |dkD  |dk  z  | dz  d      S )Ng      g      @gUUUUUU?        re   rf   ri   rx   s     r-   hardsigmoid_backwardr      s0     ;;	$y! r,   min_valmax_valc                 B    t        j                  ||k  ||k\  z  d|       S )Nr   r   )ri   rx   r   r   s       r-   hardtanh_backwardr      s$    
 ;;DGO<c;OOr,   c                 l    | t        j                  t        j                  | dz   d      d      z  dz  S r   r   r   s    r-   	hardswishr      s.     %++ekk$(:BBQFFr,   c           
      x    t        j                  |dk  dt        j                  |dk  | |dz  dz   z  |             S )Nr   r         ?r   r   s     r-   hardswish_backwardr      sA     ;;r	DAI{tax3.>?M r,   c                 6    t        j                  ||k  d|       S rp   r   )ri   rx   r`   s      r-   threshold_backwardr      s     ;;ty(![99r,   negative_slopeself_is_resultc                 <    t        j                  |dkD  | | |z        S rp   r   )ri   rx   r   r   s       r-   leaky_relu_backwardr      s      ;;taxkN.JKKr,   gradapproximatec                    d}d}d}|dk(  ri||z  dz  }d}||z  }||z  }	||||	z  z   z  }
t        j                  |
      }d|z  }d|z   }d|z  }d||z  z
  }|dd|z  |z  z   z  }||z  |z  }| ||z   z  S |}||z  dz  }ddt        j                  ||z        z   z  }|t        j                  ||z  d	z        z  }| |||z  z   z  S )
Ng;f?g;f?gmBP?tanhr   gHm?r#   r   g      )re   r   erfrd   )r   rx   r   M_SQRT2	M_SQRT1_2
M_2_SQRTPIkBetakKappax_sqx_cuberG   
tanh_innerleftrightleft_derivativetanh_derivativeinner_derivativeright_derivativekAlphacdfpdfs                        r-   gelu_backwardr      s'    %G&I'Jf*$s*d{/0ZZ&
TzJ+j:55 AF
T(9$9:/14DD)99::Y&,Q4&=112eiitd 233sTCZ'((r,   inputc                     t        j                  t        j                  |            }t        j                  |      }||z  d||z  z
  z  }| ||z   z  S rX   )re   r   Fsoftplussigmoid)ri   r   input_tanh_softplusinput_sigmoidouts        r-   mish_backwardr     sV      **QZZ%67MM%(M
-
1':=P'P#P
QC-344r,   c                 2    | t        j                  |       z  S r5   )re   r   r   s    r-   silur   
  s     %--%%%r,   c                 \    ddt        j                  |       z   z  }| |z  d|d|z
  z  z   z  S rX   )re   rd   )ri   rx   r   s      r-   silu_backwardr     s<     1uyy$''(G AG(<$<==r,   weightc                 <    t        j                  | dkD  | || z        S rp   r   )rx   r   s     r-   _prelu_kernelr     s    ;;taxv}55r,   c                 ~    t        j                  |dkD  | || z        }t        j                  |dkD  d|| z        }||fS )Nr   r   r   )ri   rx   r   
input_gradweight_grads        r-   _prelu_kernel_backwardr     sE     TAX{F[4HIJ++dQhTK-?@K$$r,   noiseloweruppertraining	generatorc                    |J |r^| dk  }t         j                  | ||      }t        j                  || |z  |       }|j	                  t        j                  ||d             |S ||z   dz  }	t         j                  | |	      S Nr   r#   r$   )r   uniformre   rf   copy_
leaky_relu)
rx   r   r   r   r   r   not_positiverF   outputr   s
             r-   rrelu_with_noiser   )  s     qyLLue,\4!8T:EKKa34%-1,t^44r,   c           
      @    | j                  t        | |||||            S r5   )r   r   )rx   r   r   r   r   r   s         r-   rrelu_with_noise_r   A  s#     ::&tUE5(IVWWr,   c                 x    |r||z
  dkD  r| j                  |      S ||z   dz  }t        j                  | |||      S )Ngư>r$   )mulr   r   )ri   rx   r   r   r   r   r   r   s           r-   rrelu_with_noise_backwardr   O  sK     EEMD(u%%%-1,''~~
 	
r,   bufferc                     |dk  }t        j                  |dd      }t        j                  |dd      }t        j                  t        j                  |             }| |||d|z   z  z  z
  z  S )Nr   r#   rO   )re   rf   rd   abs)ri   rx   r   in_negative	max_derivsignrg   s          r-   log_sigmoid_backwardr   d  sg     (KKA.I;;{Ar*D		599T?"#A)da1q5k&::;;r,   loss	reductionc                     |t         j                  j                  k(  rt        j                  |       S |t         j
                  j                  k(  rt        j                  |       S | S r5   )r"   r)   ry   re   meanr*   sum)r   r   s     r-   apply_loss_reductionr   q  sH    INN(((zz$	imm))	)yyr,   dtypec                     | t         j                  k(  rt         j                  S | t         j                  k(  rt         j                  S | t         j
                  k(  rt         j                  S y r5   )re   	complex32float16	complex64float32
complex128float64r   s    r-   to_real_dtyper   z  sK    }}	%//	!}}	%""	"}} 
#r,   targetc                 *    | |z
  dz  }t        ||      S )Nr$   )r   )rx   r   r   r   s       r-   mse_lossr     s     6MaDi00r,   c                 |    |t         j                  j                  k(  rd|j                         z  nd}|||z
  z  | z  S )N       @)r"   r)   ry   numel)ri   r   r   r   norms        r-   mse_loss_backwardr     s;     #,y~~/C/C"C3D56>"[00r,   rc   c                     | |z
  j                         }t        j                  ||k  d|dz  z  |z  |d|z  z
        }t        ||      S )Nr   r$   )r   re   rf   r   )rx   r   r   r_   r   s        r-   smooth_l1_lossr    sO     6M D;;td{C$'MD$8$t:KLDi00r,   c                    |t         j                  j                  k(  rd|j                         z  nd}||z
  }t	        j
                  |      }|| z  }t	        j                  ||k  ||z  |z  |t	        j                  |      z        S rb   )r"   r)   ry   r   re   r   rf   r   )	ri   rx   r   r   r_   r   r9   abs_x	norm_grads	            r-   smooth_l1_loss_backwardr    s{    
 "+inn.B.B!B3DvAIIaLE{"I;;AEJJqM! r,   c                 h    t        | ||||      }t        ||j                         t        ||d      S NT	copy_fromcopy_toexact_dtype)r  r   shaper   )ri   rx   r   r   r_   rT   results          r-   smooth_l1_loss_backward_outr    s3     %[$	4PFj&,,/FJDQQr,   deltac           
          |t         j                  j                  k(  rd|j                         z  nd}||z
  }t	        j
                  || k  | | z  |z  t	        j
                  ||kD  || z  |z  ||z  | z              S rb   )r"   r)   ry   r   re   rf   )ri   rx   r   r   r  r   r9   s          r-   huber_loss_backwardr    s    
 "+inn.B.B!B3DvA;;	UF
	e#AItk1E94!8k;QR r,   c                 h    t        | ||||      }t        ||j                         t        ||d      S r  )r  r   r  r   )ri   rx   r   r   r  rT   r  s          r-   huber_loss_backward_outr    s3     !dFIuMFj&,,/FJDQQr,   ignore_indextotal_weightc                    |j                         dk  rdnd}|t        j                  j                  k(  r| |z  } |j	                  |      }t        j                  ||k7  |d      }t        j                  |      }	t        j                  |	||d      }	|	j                         | j                         cxkD  rdkD  rn n| j	                  |      } |Nt        |j                               D 
cg c]  }
d }}
|j                  d   ||<   |j                  |      }| |z  } t        j                  ||k7  | d      } |	| z  S c c}
w )Nr$   r   r#   g      )rK   r"   r)   ry   rQ   re   rf   
zeros_likescatterrP   r  reshape)ri   rx   r   r   r   r  r  channel_dimsafe_targetrT   rR   	new_shapes               r-   _nll_loss_backwardr    s    xxzA~!1KINN(((!L0k*F++f4fa@K!!$'Jz;TJJ~~+//+/a/!++K8 %dhhj 121Q2	2!'a	+	*!F*++f4k1EK## 3s   *	D<c                    |j                         dkD  sJ d       t        j                  |j                         |      }|j                  |      }|dz  dk(  sJ d| d|        |dz  }|j	                  |d|      }|j	                  |||      }t        j                  |      }d|z
  |z  |z  | z  }	|| z  }t        j                  ||	g|      S )Nr   z*glu does not support 0-dimensional tensorsr$   z.Halving dimension must be even, but dimension z	 is size rc   r}   )rK   rA   canonicalize_dimsizenarrowre   r   cat)
ri   rx   rK   wrap_dimnIn	inputSize	firstHalf
secondHalfgradInputFirstHalfgradInputSecondHalfs
             r-   glu_backwardr*    s     88:>GGG>%%dhhj#6H
))H
Ca1Q	7z3%PQqIHa3IXy)<Jz2	!	!%77)CkQ  ,k999(*=>HMMr,   c           	         d|j                         cxk  rdk  sJ d        J d       |j                         dk  sJ d       |j                         dk(  xr |j                         dk(  }|sA|j                  d   |j                  d   k(  s"J d|j                   d|j                   d       |j                         dk(  s'J d	|j                   d
|j                          df       |'|j                         |j                  d   k(  sJ d       |t        j                  j
                  k(  r}|j                         dk(  rj| j                         dk(  r| j                  d   |j                  d   k(  srJ d|j                  d    d| j                          d| j                  d           | j                         dk  r| j                         dk(  sJ d| j                          t        | ||||||      S )Nr   r$   input tensor should be 1D or 2Dr#   ;0D or 1D target tensor expected, multi-target not supportedsize mismatch (got input: 
, target: ):expected total_weight to be a single element tensor, got: z (z
 elements)rO   z<weight tensor should be defined either for all or no classesz7Expected a tensor of dimension 1 and tensor.size[0] == z but got: dimension z and tensor.size[0] == z7Expected a single element grad_output tensor, but got: )rK   r  r   r"   r(   ry   r  )ri   rx   r   r   r   r  r  no_batch_dims           r-   nll_loss_backwardr3    s    
aB!BBB!BB

EDE 88:?8vzz|q'8L

1a(J	#DJJ<z&,,qIJ  1$ D
b!3!3!5 6jA' $ 	&,,.DJJrN:FEF: INN(((TXXZ1_ A%+*;*;A*>$**Q-*O 	
EdjjQRm_ U)oo/00GHYHYZ[H\G]_	
O OO"{'8'8':a'?	YD[EVEVDWX	Y? T669lL r,   c           	      >   |j                         dk(  sJ d|j                                 |j                         dk(  sJ d|j                                 |j                  d   |j                  d   k(  r>|j                  d   |j                  d   k(  r|j                  d   |j                  d   k(  s!J d|j                   d	|j                          |j                         dk(  s&J d
|j                   d|j                          d       t        | ||||||      S )N   zSonly batches of spatial inputs supported (4D tensors), but got input of dimension: r   zUonly batches of spatial targets supported (3D tensors) but got targets of dimension: r   r$   r#   r.  r/  r1  z ( z, elements))rK   r  r   r  )ri   rx   r   r   r   r  r  s          r-   nll_loss2d_backwardr6  I  sE    	
aj	\]a]e]e]g\hij 	

n	^_e_i_i_k^lmn 	

1a(JJqMV\\!_,JJqMV\\!_,I 
$DJJ<z&,,H	I	-
 1$ 	""#3|'9'9';&<K	I$
 T669lL r,   c           	      "   |dz
  t        j                  t        j                  |        | j                  dd            z  |t        j                  t        j                  |       | j                  dd            z  z
  }|||z  }t        ||      S )Nr#   r+   i)re   maximumlog1pnew_fulllogr   )rx   r   r   r   r   s        r-   binary_cross_entropyr<  l  s     QJ%--TEDMM"d3 uyyb$0GHHID f}i00r,   c                     d}| ||z
  z  t        j                  |d|z
  z  |      z  }|||z  }|t        j                  j                  k(  r||j                         z  }|S )Ng-q=r#   r   )re   r   r"   r)   ry   r   )ri   rx   r   r   r   EPSILONr  s          r-   binary_cross_entropy_backwardr?    sg     GD6M*U[[T9JPW-XXF&INN((($**,&Mr,   c                 r    t        j                  t        j                  |  |z              }t        ||      S r5   )re   r9  rd   r   )r   r   r   r   s       r-   soft_margin_lossrA    s.     ;;uyy%&12Di00r,   c                     || z  t        j                  ||z        dz
  z  }|t        j                  j                  k(  r||j                         z  }|S rX   )re   r   r"   r)   ry   r   )ri   rx   r   r   rT   s        r-   soft_margin_loss_backwardrC    sM     +%v})E)IJJINN((($**,.
r,   otherpc                 6    t         j                  | |z
  |      S )N)rE  )r   r   )r   rD  rE  s      r-   distrG    s     99UU]a9((r,   x1x2c                    | j                  d      j                  dd      }t        j                  |t        j                        }|j                  d      j                  dd      }t        j                  |t        j                        }t        j
                  | j                  d      ||gd      }t        j
                  |||gd      }|j                  |j                        }|j                  d      j                         S )Nr$   rO   Tmemory_formatr   )powr   re   	ones_likecontiguous_formatr"  r   matmulmT	clamp_minsqrt)	rH  rI  x1_normx1_padx2_normx2_padx1_x2_r  s	            r-   _euclidean_distr[    s     ffQimmB%G__WE4K4KLFffQimmB%G__WE4K4KLF
))RVVBZ&12
6C
))R)2
.CZZFA##%%r,   input_sizesstartendstepc                 X    | j                  |      }t        j                  || ||||      S r5   )	new_zerosre   slice_scatter)ri   r\  rK   r]  r^  r_  rT   s          r-   slice_backwardrc    s/     &&{3Jz;UCNNr,   r#   c                    | j                         }|dk(  rt        d      t        j                  | j                         |      }t	        | j                               }t	        | j                               }|dk  rt        d      ||nd}||nt        j                  }	|dk  r|||   z  }|	dk  r|	||   z  }	|dk  rd}n|||   kD  r||   }|	|k  r|}	n|	||   kD  r||   }	| j                         |||   z  z   }
|	|z
  }||z   dz
  |z  ||<   ||xx   |z  cc<   | j                  rt        d      | j                  |||
      S )Nr   z,slice() cannot be applied to a 0-dim tensor.zslice step must be positiver#   z<Slice decomposition for quantized tensors aren't implemented)rK   RuntimeErrorrA   r  listr   stridesysmaxsizestorage_offsetis_quantizedNotImplementedError
as_strided)rx   rK   r]  r^  r_  ndimsizesstrides	start_valend_valrj  lens               r-   slice_forwardrt    sm    88:DqyIJJ

 
 S
1CE4;;=!Gqy899*I_c#++G1}U3Z	{5:1}		U3Z	#J		5:	*((*Y-EEN
I
C*q.T)E#JCLDL!J
 	
 ug~>>r,   indexc                 T    | j                  |      }t        j                  || ||      S r5   )ra  re   select_scatter)ri   r\  rK   ru  rT   s        r-   select_backwardrx    s+     &&{3J
KeDDr,   offsetdim1dim2c                 V    | j                  |      }t        j                  || |||      S r5   )ra  re   diagonal_scatter)ri   r\  ry  rz  r{  rT   s         r-   diagonal_backwardr~    s-    
 &&{3J!!*k64NNr,   input_dtypec                 F    | j                   |k7  r|j                  |      }|S r5   )r   r8   )ri   rT   r  s      r-   _cast_grad_to_input_dtyper    s&     K']];/
r,   r   c                 ~    | |z  }||t        j                  ||d      z  z
  }t        | ||      j                         S NTrK   keepdim)re   r   r  
contiguous)ri   r   rK   r  new_grad_outputrT   s         r-   _softmax_backward_datar    sK     "F*O 6EIIS$- $ J %[*kJUUWWr,   c                 ~    | t        j                  |      t        j                  | |d      z  z
  }t        | ||      S r  )re   rd   r   r  )ri   r   rK   r  rT   s        r-   _log_softmax_backward_datar  .  sA     uyy0599d4   J %[*kJJr,   c                     | |dz  z   ||dz
  z  z
  }t        t        j                  t        j                  |      } |d||      j	                  d      } |d||z  |      j	                  d      }	||	z   S )z/Utility function to implement im2col and col2imr$   r#   r   devicer   rO   )r   re   arangeint64rQ   )
input_dkernel_d
dilation_d	padding_dstride_dr  blocks_d	arange_kwblocks_d_indiceskernel_grids
             r-    _im2col_col2im_indices_along_dimr  :  s     Q&x!|)DDHEKKGI !Hh7AA!D Ax*4jAKKBOK k))r,   kernel_sizedilationpaddingrg  c           
         t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        dd} |d        |d        |d	d
        |d       | j                  t              }t        j                  |dv xr t	        d dd  D              fd       t        d t        dd        D              t        j                  t	        d D              fd       |dk(  }|s| j                  d      } | j                  \  }}	}
}\  }}\  }}\  }}\  }}t        |
||||| j                        }t        |||||| j                        }t        j                  | ||||f      }|j                  d      j                  d      }|d d d d ||f   }|j                  dddddd      }|j                  d      }|j                  d      }|j                  ||	|z  |z  ||z        }|s|j                  d      }|S )Nr$   c                       y)Nz"im2col(): only 2D kernel supportedr+   r+   r,   r-   r~   zim2col.<locals>.<lambda>V      r,   c                       y)Nz$im2col(): only 2D dilation supportedr+   r+   r,   r-   r~   zim2col.<locals>.<lambda>W  r  r,   c                       y)Nz#im2col(): only 2D padding supportedr+   r+   r,   r-   r~   zim2col.<locals>.<lambda>X  r  r,   c                       y)Nz"im2col(): only 2D stride supportedr+   r+   r,   r-   r~   zim2col.<locals>.<lambda>Y  r  r,   c                 ~    |rt        d | D              nt        d | D              }t        j                  |d        y )Nc              3   &   K   | ]	  }|d kD    ywr   Nr+   .0rE  s     r-   	<genexpr>z1im2col.<locals>.check_positive.<locals>.<genexpr>\       (Q1q5(   c              3   &   K   | ]	  }|d k\    ywr  r+   r  s     r-   r  z1im2col.<locals>.check_positive.<locals>.<genexpr>\       ;RqAF;Rr  c                       y)Nz<{param_name} should be greater {'than' zero, but got {param}r+   r+   r,   r-   r~   z0im2col.<locals>.check_positive.<locals>.<lambda>^  r  r,   allre   r   param
param_namestrictconds       r-   check_positivezim2col.<locals>.check_positive[  s3    ,2s(%((;RE;R8RX	
r,   r  r  r  Fr  rg  r   r5  c              3   &   K   | ]	  }|d k7    ywr  r+   r  ds     r-   r  zim2col.<locals>.<genexpr>i       :!qAv:r  r   c                       dt                S )NzmExpected 3D or 4D (batch mode) tensor for input with possible 0 batch size and non-zero dimensions, but got: tupler  s   r-   r~   zim2col.<locals>.<lambda>j       --25\N< r,   c              3   \   K   | ]$  \  }}}}}d |d|z  z   ||d z
  z  z
  d z
  |z  z    & yw)r#   r$   Nr+   )r  r   paddilkersts         r-   r  zim2col.<locals>.<genexpr>m  sF      "Cc3 	
S1s7]SC!G_,q0R77s   *,rM  c              3   &   K   | ]	  }|d kD    ywr  r+   )r  cs     r-   r  zim2col.<locals>.<genexpr>t  s     'aAE'r  c                  F    dt        dd         d d  d d d dS )	Nz!Given an input with spacial size rM  , kernel_size=, dilation=
, padding=	, stride=z9, the calculated shape of the array of sliding blocks is z*, but its components must be at least one.r  )r  r  output_sizer  r  rg  s   r-   r~   zim2col.<locals>.<lambda>u  sL    3E%*4E3F G"m;xj 9)9VH -]D	F r,   r5  r   rO   r#   r      T)re   r   rs  r  r  r  ziprQ   r  r  r   r  permuter   r  squeeze)r   r  r  r  rg  r  rn  batched_input	batch_dimr  input_hinput_wstride_hstride_w	padding_h	padding_w
dilation_h
dilation_wkernel_hkernel_wblocks_row_indicesblocks_col_indicespadded_inputr   num_blocks_rownum_blocks_colr  r  s    ````                     @@r-   im2colr  M  s~    
LL[!Q&(TU	LLX!#%ST	LLW"$QR	LLV!#OP
 ;.8Z(8Yu568$KKEu:D	LL:3:uRSz::	<
  &)"#J;'
 K 
LL';''	F 	F AIM"/4{{,I{GWHh"Iy%J
$Hh9:y(ELL ::y(ELL 55Iy) LML+55b9CCBG!Q 24FFGF^^Aq!Q1-F',,Q/N',,Q/N^^;)H4n~6UF "Mr,   r  c                   !" t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        dd} |d	        |d
        |dd        |d        |d       | j                  "t        "      }t        j                  |dv xr t	        d "dd  D              "fd       d   d   z  }t        j                  "d   |z  dk(  "fd       t              D 	
cg c]"  \  }	}
}}}d|	d|
z  z   ||dz
  z  z
  dz
  |z  z   $ }}}}
}	}|d   |d   z  !t        j                  "d   !k(  !"fd       t        j                  !dkD  !"fd       |dk(  }|s| j                  d      } | j                  "\  }}\  }}\  }}\  }}\  }}| j                  "d   "d   |z  gz   |z         } | j                  dddddd      } t        |||||| j                        }t        |d      }t        |||||| j                        }t              D cg c]  \  }}|d|z  z    }}}| j                  "d   "d   t              z  g|z         }d d ||f} t        j                  || | d      }t!        j"                  || | | | f      }|s|j%                  d      }|S c c}}}}
}	w c c}}w )Nr$   c                       y)Nzonly 2D output_size supportedr+   r+   r,   r-   r~   zcol2im.<locals>.<lambda>  r  r,   c                       y)Nzonly 2D kernel supportedr+   r+   r,   r-   r~   zcol2im.<locals>.<lambda>  r  r,   c                       y)Nzonly 2D dilation supportedr+   r+   r,   r-   r~   zcol2im.<locals>.<lambda>  r  r,   c                       y)Nzonly 2D padding supportedr+   r+   r,   r-   r~   zcol2im.<locals>.<lambda>  r  r,   c                       y)Nzonly 2D stride supportedr+   r+   r,   r-   r~   zcol2im.<locals>.<lambda>  r  r,   Tc                 ~    |rt        d | D              nt        d | D              }t        j                  |d        y )Nc              3   &   K   | ]	  }|d kD    ywr  r+   r  s     r-   r  z1col2im.<locals>.check_positive.<locals>.<genexpr>  r  r  c              3   &   K   | ]	  }|d k\    ywr  r+   r  s     r-   r  z1col2im.<locals>.check_positive.<locals>.<genexpr>  r  r  c                       y)Nz9{param_name} should be greater than zero, but got {param}r+   r+   r,   r-   r~   z0col2im.<locals>.check_positive.<locals>.<lambda>  r  r,   r  r  s       r-   r  zcol2im.<locals>.check_positive  s3    ,2s(%((;RE;R8RU	
r,   r  r  r  Fr  rg  r  )r$   r   c              3   &   K   | ]	  }|d k7    ywr  r+   r  s     r-   r  zcol2im.<locals>.<genexpr>  r  r  rM  c                       dt                S )NzmExpected 2D or 3D (batch mode) tensor for input with possible 0 batch size and non-zero dimensions, but got: r  r  s   r-   r~   zcol2im.<locals>.<lambda>  r  r,   r   r#   c                      dd    d  S )Nz|Expected size of input's first non-batch dimension to be divisible by the product of kernel_size, but got input.shape[-2] = rM  z and kernel_size=r+   )r  r  s   r-   r~   zcol2im.<locals>.<lambda>  s#     ==B2YK H"m% r,   rO   c                  :    d d d d d d  dd    d	S 
NzGiven output_size=r  r  r  r  z , expected input.size(-1) to be 	 but got rO   .r+   Lr  r  r  r  r  rg  s   r-   r~   zcol2im.<locals>.<lambda>  F    $[M} M:Zy	& B))*9U2YKqB r,   c                  :    d d d d d d  dd    d	S r  r+   r  s   r-   r~   zcol2im.<locals>.<lambda>  r  r,   r   r5  r  
accumulater  )re   r   rs  r  r  r  rQ   r  r  r  r  rS   ra  prodr   _unsafe_index_putr   r  r  )#r   r  r  r  r  rg  r  rn  prod_kernel_sizer   r  r  r  r  colr  out_hout_wr  r  r  r  r  r  r  r  indices_rowindices_colorE  output_padded_sizer   idxr  r  s#    `````                           @@r-   col2imr    s    
LL[!Q&(OP	LL[!Q&(JK	LLX!#%IJ	LLW"$GH	LLV!#EF
 ;.8Z(7Ie468$;.KKEu:D	LL:3:uRSz::	<
 #1~A6	LLb	$$)	% '*(K'
 "Cc3 	
S1s7]SC!G_,q0R77C  	AQA	LLb	Q	B 	B 
LL	A	B 	B AIM"KKELE5Hh"Iy%J
$Hh MM58U1X1A%AB[PSVVWEMM!Q1a+E2xY%,,K $K3K2xY%,,K 14K0IJ1!a!e)JJ__	q58tK0014FFF {K
0C##FC4#HFUU6YJ
YJ
KLF"MkV Ks   6'M9Mmaskc                 z    | |j                  |       |z  z  j                  t        j                  |             }|S )NrK  )type_asclonerA   suggest_memory_format)ri   r  rk   rF   s       r-   native_dropout_backwardr    sB     
[1E9	:AA11+> 	B 	A Hr,   
input_size	dimensionr   c                    t        |      dk(  rt        j                  | d      S t        j                  t        |      |      }t        j
                  ||   | j                  t        j                        }|j                  d||      j                         }| j                  d|dz         j                  ||dz         } | j                  |      }d|z  |fz   }t        j                  ||| d      j                         S )Nr   r  r   rO   r#   r5   Tr  )rs  re   squeeze_copyrA   r  r  r  int32unfoldflattenmovedimra  r   r  r  )	r   r  r  r   r_  rK   r  rT   ru  s	            r-   unfold_backwardr    s    
 :!!!$**

 
 Z)
<C
,,z#t{{%++
NC
**Qd
#
+
+
-C<<C!G$,,S#':D 
+JcMSF"E!!*eTd!KVVXXr,   epsc           
      .   |A|}d|z
  }t        j                  t        j                  ||k\  ||k        | |d|z
  z  z  d      S t        j                  t        j                  |dk\  |dk        | |d|z
  z  z  |j                  dt	        d                  S )Nrc   r   r+   nan)re   rf   logical_andr:  float)ri   rx   r  lohis        r-   logit_backwardr  $  s    
 2X{{dbj$"*543:./
 	
 {{dck43;743:./MM"eEl+
 	
r,   trainc                 d    |r|dk7  rt         j                  | ||      d   S | j                         S rp   )r   native_dropoutr  )r   rE  r   s      r-   dropoutr#  9  s3     a""5!U3A66{{}r,   out0out1c                    |r|dk7  r|dk(  r:t        j                  |       t        j                  | t         j                        fS | j                  j                  st        d      t        j                  |       |kD  }|| z  t        dd|z
  z        z  }||fS | t        j                  | t         j                        fS )Nr   r#   r   z?result type Float can't be cast to the desired output type Longrc   )	re   r  boolr   is_floating_pointre  	rand_liker  rO  )r   rE  r   	bool_maskress        r-   r"  r"  C  s     a6$$U+U-=-=e5::-VWW{{,,Q  OOE*Q.	%%sQw"88YuuEJJ?@@r,   half_to_floatc                    | j                         } |r| j                  t        j                  k(  sJ t	        j
                  | t        j                  j                        \  }}| j                  |      } | j                         dk(  rt        j                  |       }n0t        j                  | |d      }t        j                  | |z
        }|t        j                  ||d      z  }|s|j                  |      }|S Nr3   r   Tr  )r  r   re   halfrA   rB   ELEMENTWISE_TYPE_PROMOTION_KINDDEFAULTr8   r   rd   amaxr   )r9   rK   r,  r:   r=   unnormalizedx_maxr  s           r-   _softmaxr7  T  s    
 	
Aww%**$$$&+&>&>	uDDLL'#| 	
AwwyA~yy|

1c40yyU+EIIlCFFF<(Mr,   c                     | j                         } |r| j                  t        j                  k(  sJ t	        j
                  | t        j                  j                        \  }}| j                  |      } | j                         dk(  r| }nt        j                  | |d      }| |z
  }t        j                  t        j                  t        j                  |      |d            }||z
  }|s|j                  |      }|S r.  )r  r   re   r1  rA   rB   r2  r3  r8   r   r4  r;  r   rd   )	r9   rK   r,  r:   r=   shiftedr6  shifted_logsumexpr  s	            r-   _log_softmaxr;  k  s    
 	
Aww%**$$$&+&>&>	uDDLL'#| 	
AwwyA~

1c40e)		%))EIIg,>T"RS((F<(Mr,   indicespadding_idxscale_grad_by_freqsparsec                     | j                         dk(  sJ d       |j                  dk  r4| j                  d|      }|j                  dk(  r|j                  d      }|S | |   S )Nr$   z'weight' must be 2-Dr#   r   )rK   rn  index_selectr  )r   r<  r=  r>  r?  r   s         r-   	embeddingrB    sd     ::<1444||q!!!W-<<1++a.C
gr,   num_weightsc                 t   t        j                  | t         j                  j                        \  }}| j	                  |      } t        |t        j                        }|rZ|j                  |f      }t        j                  |      }t        j                  ||g|d      }||   }	| |	j                  d      z  } t        ||k(  | j                        }
| j                  |
d      }| j                  |f| j                   |j                  d  z         }t        j                  ||g|d      j	                  |      S )Nr/  Tr  rO   r   )rA   rB   r2  r3  r8   r   re   longra  rO  r   r  rQ   rS   rn  masked_fillr  )ri   r<  rC  r=  r>  r:   r=   countsonesgrad_weights_scaler  r   grad_weights                r-   embedding_dense_backwardrK    s,    ',&>&>)N)N)V)V'#| ..!23K%guzz:G""K>2w'''	4D'Q#G_!$6$@$@$DDW3[5E5EFD""4+D''	**7<<>::K !!+y$4!PSS r,   c                 "    d}| D ]  }||z  }	 |S rX   r+   )r9   rF   is      r-   r  r    s$    	A 	QHr,   tensors
num_chunksc                 Z   g }| D ]  }|j                         }||   |z   dz
  |z  |z  }|||   k7  r;dgdz  |j                  |z
  dz
  z  d|||   z
  gz   }t        j                  ||d      }|d | t	        j
                  |dg      z   }|j                  |j                  |              |S )Nr#   r   r$   rO   )r   rn  r   constant_pad_ndre   Sizeappendview)	rN  rK   rO  padded_tensorstensortensor_sizepad_along_dimr  	view_sizes	            r-   
_pad_chunkrZ    s    
 N 6kkm$S)J6:zIJVK,,#'V[[3.23C 007 C ))&#q9F%

J3C(DD	fkk)456 r,   c                 R    | d   j                   }| D ]  }|j                   |k7  s y y)Nr   FTrn  )rN  rn  rV  s      r-   have_same_ndimsr]    s2    1:??D ;;$ r,   c                     | d   j                         d | }| D ]-  }t        j                  |j                         d | |k(  d        / y )Nr   c                       y)NzG_chunk_cat expects same sizes of 0,...,dim-1 dimensions for all tensorsr+   r+   r,   r-   r~   z+leading_dimension_matches.<locals>.<lambda>  r  r,   )r   re   r   )rN  rK   leading_dim_sizesrV  s       r-   leading_dimension_matchesra    sN    
)$3/ 
KKM$3#44]	

r,   c                    t        j                  |dk\  d        t        j                  t        |       dkD  d        | d   j                  }| d   j                  }| D ]r  }t        j                  |j                         dkD  d        t        j                  |j                  |k(  d        t        j                  |j                  |k(  d        t t        |       r(t        j                  | d   j                         |      }nEt        j                  |dk\  d        | D ]&  }t        j                  ||j                  k  d	        ( t        | |       |S )
Nr#   c                       y)Nz&_chunk_cat expects positive num_chunksr+   r+   r,   r-   r~   z._preprocess_chunk_cat_inputs.<locals>.<lambda>  r  r,   r   c                       y)Nz0_chunk_cat expects a non-empty input tensor listr+   r+   r,   r-   r~   z._preprocess_chunk_cat_inputs.<locals>.<lambda>  r  r,   c                       y)Nz#_chunk_cat expects non-empty tensorr+   r+   r,   r-   r~   z._preprocess_chunk_cat_inputs.<locals>.<lambda>  r  r,   c                       y)Nz8_chunk_cat expects all input tensors with the same dtyper+   r+   r,   r-   r~   z._preprocess_chunk_cat_inputs.<locals>.<lambda>  r  r,   c                       y)Nz8_chunk_cat expects all inputs tensors on the same devicer+   r+   r,   r-   r~   z._preprocess_chunk_cat_inputs.<locals>.<lambda>  r  r,   c                       y)NzK_chunk_cat expects non-negative dim when input tensors have different ndimsr+   r+   r,   r-   r~   z._preprocess_chunk_cat_inputs.<locals>.<lambda>  r  r,   c                       y)Nz3_chunk_cat expects dim < ndim for all input tensorsr+   r+   r,   r-   r~   z._preprocess_chunk_cat_inputs.<locals>.<lambda>  r  r,   )re   r   rs  r   r  r   r]  rA   r  rK   rn  ra  )rN  rK   rO  expected_dtypeexpected_devicerV  s         r-   _preprocess_chunk_cat_inputsrl    s-   
 
LLq"RS	LLGqT QZ%%Naj''O 	
V\\^a')VWLLN*N	
 	MM_,N	
	
 w$$WQZ^^%5s;1Ha	
  	FLLfkk!M	
 gs+Jr,   r   c                     t        | ||      }t        | ||      }|t        j                  ||dz         S t        j                  ||dz   |       |S )Nr#   )r   )rl  rZ  re   r"  )rN  rK   rO  r   rU  s        r-   
_chunk_catrn  	  sS     'wZ
@Cj9N
{yyq11		.#'s3
r,   split_sizesc                     t        t                    D ]  }t        j                  |   d         t        j                  t
        t               j                     k(   fd       t              }g }d}ddlm	} t        |      D ]I  }|   } |||z    j                     k         |j                   j                  ||             ||z  }K |S )Nc                       y)NzCsplit_with_sizes expects split_sizes have only non-negative entriesr+   r+   r,   r-   r~   z"split_with_sizes.<locals>.<lambda>"  r  r,   c                  @    dt               dj                       S )NzSplit sizes add up to z but got the tensor's size of )r   r  )rK   rx   ro  s   r-   r~   z"split_with_sizes.<locals>.<lambda>'  s*    ([)9(::XY]YcYcdgYhXij r,   r   )expect_true)rP   rs  re   _check_is_size_check_with
ValueErrorr   r  %torch.fx.experimental.symbolic_shapesrs  rS  r!  )	rx   ro  rK   rM  
num_splitssplits	start_idxrs  lengths	   ```      r-   split_with_sizesr|    s     3{#$ 
NY	


 
KDJJsO+j
 [!JFI B: Q 	I&$**S/9:dkk#y&9:V	 Mr,   c                     t        | ||      }|.|D cg c]"  }|j                  t        j                        $ c}S t	        ||      D ])  \  }}t        ||j                         t        ||d       + y c c}w )Nr}   rK  Tr  )r|  r  re   rP  r  r   r  r   )rx   ro  rK   r   ry  sr   splits           r-   split_with_sizes_copyr  ;  sz     dKS9F
{HNO1e&=&=>OO f- 	NMFEfekk2UFM	N  Ps   'A8
split_size.c                 D    t         j                  j                  | ||      S r5   )r   r  r   )r   r  rK   s      r-   unsafe_splitr  N  s    ::UJ44r,   c                 D    t         j                  j                  | ||      S r5   )r   r|  default)r   ro  rK   s      r-   unsafe_split_with_sizesr  S  s       ((SAAr,   c                     | j                   }||   }|dk(  r
|dk(  sJ | fS ||z   dz
  |z  }ddlm}  ||      }t        |      D cg c]  }| }}|||z  |z
  z
  |d<   t	        j
                  | ||      S c c}w )Nr   r#   )	guard_intrO   )r  rw  r  rP   re   r  )	rx   r  rK   r\  dim_sizechunksr  rM  ro  s	            r-   r  r  Z  s    **K3HQ1}}w#a'J6F @vF',V}5!:5K5 J$7($BCKO;;t[#.. 6s   	A7tensor_indices_or_sectionsc                    |j                   j                  dk(  sJ |j                  t        j                  k(  sJ |j                         t        j                  dk(  xs dk(  fd       dk(  r4|j                         }t        |t              sJ | j                  ||      S |D cg c]  }|j                          }}| j                  ||      S c c}w )Ncpur#   r   c                      d  dS )Nz{tensor_split expected tensor_indices_or_sections to be a zero-dimensional or one-dimensional tensor, but got a tensor with z dimsr+   )	split_dims   r-   r~   zAtensor_split_tensor_indices_or_sections_py_impl.<locals>.<lambda>y  s     <<E;eM r,   )r  typer   re   r  rK   r   itemr7   r   tensor_split)rx   r  rK   sectionsrM  r<  r  s         @r-   /tensor_split_tensor_indices_or_sections_py_implr  l  s     &,,11U:::%++u{{:::*..0I	LLQ()q.	M
 A~-224(G,,,  3//%?@1668@@  #.. As   ,Cmat1mat2c                     | j                         s&| j                         st        |      }t        |      }|t        j                  ||      z  }|dk(  r|S ||| z  z   S rp   )r(  
is_complexintre   mm)rx   r  r  r_   rj   r   s         r-   addmmr    s]     !!#DOO,=4yE

%((4&
&Cqy
 r,   use_geluc                     t        | ||||      }|r8| j                  rt        j                  |d      S t        j                  |      S t        j	                  |      S )Nr   )r   )r  is_cudar   gelurelu)rx   r  r  r_   rj   r  r   s          r-   _addmm_activationr    sO     dD$
.C<<99Sf95599S>!99S>r,   vecc                     | j                         s&| j                         st        |      }t        |      }|t        j                  ||      z  }|dk(  r|S ||| z  z   S rp   )r(  r  r  re   mv)rx   r  r  r_   rj   r   s         r-   addmvr    s[     !!#DOO,=4yE

%((4%
%Cqy
r,   r   rstdgammaNCHxWgroupoutput_maskc
           	      R   t        j                  | ||d       t        j                  || d       t        j                  |d       t        j                  |j                         z  z  k(  fd       t        j                  j                  fk(  fd       t        j                  d u xs j                         k(  fd       t              \  }
}t        j                  |dk(  fd       t        j                  | |      j                        j                  dg	      }| j                        j                  dg	      }d }d }d }|	d   r*d
|
z  z  }t        j                  |j                  d            j                  |
      j                  d      }t        j                  |j                  d            j                  |
      j                  d      }t        j                  |j                  d      j                  d|
            }n|j                  |
      j                  d      }|j                  |
      j                  d      }t        j                  |j                  d      t        j                  d|
f|j                              }|z  |z
  |z  |z  |z  |z  }| z  ||z  |z  z
  }|j                  d      }t        |d      }t        |d      }t        j                  | j                  |
      |      t        j                  |j                  |
      |      z   |z   }|j                  |j                        j!                  |j"                        }|	d   rk|j                  |
      |j                  |
      j                  d      z  z
  |j                  d      z  j                  dg	      j                        }|	d   r|j                  dg	      }|||fS )NF)allow_cpu_scalar_tensorsc                      d z  z   dS )NzExpect input to have z	 elementsr+   )r  r  r  s   r-   r~   z,native_group_norm_backward.<locals>.<lambda>  s    'Q~Y? r,   c                  .    d  d dj                    S )NzExpect mean to have shape (, z
, but got r  )r  r  r   s   r-   r~   z,native_group_norm_backward.<locals>.<lambda>  s    -aS5'DJJ<P r,   c                  <    d  dj                          S d S )NzExpect gamma to have z elements but got rO   )r   )r  r  s   r-   r~   z,native_group_norm_backward.<locals>.<lambda>  s-    's*<eN_U[[]<hi eg<hi r,   r   c                      d  d S )NzExpect number of channels z, to be evenly-divisible by number of groups r+   )r  r  s   r-   r~   z,native_group_norm_backward.<locals>.<lambda>  s    ,QC/[\a[bc r,   r$   r}   rc   rO   r#   r  r5  )rA   check_same_devicecheck_same_shapere   r   r   r  divmodr   rT  r   rQ   r  rH  r  rS   r8   r   )ri   r   r   r  r  r  r  r  r  r  cpg_remdsdbd_inputd_gammad_biasr~  ds_valdb_valc1c2c3s     ` `````              r-   native_group_norm_backwardr    s    
UD$ 
5+N	4F	LLQ$? 
LL

q%j P 
LL+!+i
 q% IC	LL	c 
;	&	+	+Aq#	6	:	:s	:	CB			!Q	$	(	(aS	(	1B $G $G#F1~39YYr5??1#56>>q%MQQRSTFYYr5??1#56>>q%MQQRSTFr"a,B
 ZZ5#.2215FZZ5#.2215Fr"

Auc?4;;?B tmf$,t3d:Q>S4Z&4-!++\\"r1%r1%IIk))!UC=rBiiaS92>? 	
 //%++.11%++>1~ E3'"''!UC*@4>>RTCU*UU..$% SaSS\WQZ 	 1~QCWf%%r,   out2c
                    t        | |||||||||	
      }|
||f}t        |      D ]2  \  }}|	t        ||   |j                         t	        |||   d       4 |S r  )r  	enumerater   r  r   )ri   r   r   r  r  r  r  r  r  r  r$  r%  r  r  rT   rM  rF   s                    r-   native_group_norm_backward_outr    sz    " (UD$q!S%F d#J&! Q1=jmQWW5Q
14PQ
 r,   c                 ,    | | j                  |      S | S r5   r8   )r9   r   s     r-   _maybe_castr  1  s    }ttE{Hr,   grad_outnormalized_shapebiasc                   ! |j                   }|j                         }	t        j                  |j                        !!fd| |||fD        \  }
}}}|
J |	t        |      z
  }||d  }|d | }g }g }t        |	      D ]*  }||k\  r|j                  |       |j                  |       , t        |      }t        |      }|dk  s|dk  rN|d   r|j                  |      nd |d   r|j                  ||d        nd |d   r|j                  ||d        fS d fS t        ||j                               }t        ||j                               }||z
  |z  }||
|z  }n|
}||z  }t        j                  ||d      }t        j                  ||      }t        j                  ||d      }t        j                  ||      }||z
  |z
  }d }d }d } |d   r||z  |z  }|d   r0|.t        |      dkD  rt        j                  |
|z  |d      }n|
|z  }|d   r8|6t        |      dkD  rt        j                  |
|d      } n|
j                         } t        ||j                        t        ||j                        t        | |j                        fS )Nc              3   d   K   | ]'  }||j                        j                         n| ) y wr5   )r8   r  r  r9   r:   s     r-   r  z-native_layer_norm_backward.<locals>.<genexpr>F  s5      9 12**,1D9s   -0r   r#   r$   TF)r  rK   rA   get_computation_dtyper   rs  rP   rS  r  ra  rS   re   r   r   r  r  )"r  r   r  r   r  r   r  r  input_shape
input_ndimgrad_out_cast
input_castweight_cast	bias_castaxis
inner_dims
outer_dimsinner_dim_indicesouter_dim_indicesrM  r  Mx_hat
grad_x_hatabr  r  r  rG   r  d_weightr  r:   s"                                    @r-   native_layer_norm_backwardr  8  s    ++KJ33EKK@9E64095M:{I $$$,--DTU#JUd#J#%#%: (9$$Q'$$Q'	( 	ZAZAAva,7NEOOK(3>q>EOOK./t3>q>EOOK./
 	
 HL
 	

 T:>>#34DT:>>#34D$$&E"[0
"
QA		*/6A	:u	%B	2($	/B	5"	BEBJE $G!%H#F1~!8u$1~+1 !A%yy!68I5QH$u,H1~)/ !A%YY}.?GF"((*F 	GU[[)Hekk*FEKK( r,   c          
          t        | |||||||      }||	|
f}t        |      D ]2  \  }}|	t        ||   |j                         t	        |||   d       4 |S r  )r  r  r   r  r   )r  r   r  r   r  r   r  r  r$  r%  r  r  rT   rM  rF   s                  r-   native_layer_norm_backward_outr    sw     (%)4vt[F d#J&! Q1=jmQWW5Q
14PQ
 r,   running_meanrunning_varmomentum
functionalc	                 2   dgt        t        d| j                                     z   }	t        j                  | j
                        }
|}|}|r"t        j                  | j
                        }
| j                  |
      }t        j                  ||	dd      \  }}t        j                  ||z         }| |z
  |z  }t        j                  ||	      }t        j                  ||	      }|!||z  d|z
  |z  z   }|s|j                  |       |2| j                         | j                  d   z  }t        j                  ||	      }|||dz
  z  z  }||z  d|z
  |z  z   }|s|j                  |       n||J |j                  |
d      }|}|j                  |
d      }|}|}dt        j                  ||z         z  }| j                  j                   dk7  r|}|}n"| j#                  d	      }| j#                  d	      }t%        || j                         dz
        }t%        || j                         dz
        }| |z
  |z  }|2|j'                         }t%        || j                         dz
        }||z  }|2|j'                         }t%        || j                         dz
        }||z   }| j                  j                   dk(  r8|j                  | j
                        }|j                  | j
                        }|j                  | j
                        ||||fS )
Nr   r$   r   T)rK   
correctionr  r#   )r   r   r  r   )rf  rP   rK   rA   r  r   r8   re   var_meanrsqrtr  r   r   r  rT  r  r  ra  rS   r  )r   r   r  r  r  r   r  r  r  reduction_dimsr:   new_running_meannew_running_var	input_acc
biased_varr   r  r   	save_mean	save_rstdnsqueezed_varunbiased_varinvstds                           r-   native_batch_norm_helperr    s    S4a 566N33EKK@#!O!77DHH#4H5	 >>>a

D {{:+,$,$&MM$7	MM$7	#')3q8||6SS""#34"A.A !==^DL'1A;7L&5X8TTO!!/2'K,CCC#->TJ'!nn+<4nH%ejjs!234<<%$II-I-I uyy{Q7"6599;?;$,&(!"6599;?;&||~ uyy{Q7$||E!LLu{{L3	LLu{{L3				$ r,   r  save_invstdc                 >    t        | |||||||d	      \  }}	}
}}||	|
fS NFr  r   r   r  r  r  r   r  r  r   r  r  rR   s               r-   native_batch_normr    s=     *Bvt\;(CQV*&FIy!Q 9i''r,   c           
          ||t         j                  | |||||      S |t        d      |t        d      |rt         j                  | |||||||      S t         j                  | ||||||      S )Nz`running_mean is None, but running_var is provided. They should both be None or both be provided.z`running_var is None, but running_mean is provided. They should both be None or both be provided.)r   _native_batch_norm_legitre  $_native_batch_norm_legit_no_training)r   r   r  r  r  r   r  r  s           r-   native_batch_norm_decompositionr    s      3,,648S
 	
 <
 	
 <
 	
 ,,64{HhPS
 	
 8864{Hc
 	
r,   c                 l   | j                  |      }||z   dz
  |z  }|dk(  rZ|dk(  rU|D cg c]  }| }}|||z  |z
  z
  ||dz
  <   t        j                  j                  j                  j                  | ||      S t        j                  j                  j                  j                  | ||      S c c}w Nr#   r   )r   re   opsr   r  r  r  r   )rV  r  rK   r  r  rR   ro  s          r-   unsafe_chunk_py_implr	  5  s    {{3HV#a'F2JQ8q=+12az22",
V0Ch0N"OFQJyy~~55==fkSVWW99>>&&--fj#FF 3s   	B1c           
      N    t         j                  j                  | ||||d||      S r  )r   r  r  )r   r   r  r  r  r  r  s          r-   r  r  A  s5     ((00	 	r,   c                 >    t        | |||||||d	      \  }}	}
}}||	|
fS r  r  r   s               r-   r  r  W  s=     *Bvt\;(CQV*&FIy!Q 9i''r,   c                 >    t        | ||d d |||d	      \  }}}}	}	|||fS r  r  )
r   r   r  r   r  r  r   r  r  rR   s
             r-   !_native_batch_norm_legit_no_statsr  h  s<     *BvtT48S%*&FIy!Q 9i''r,   c                 f    t        | |||||||d	      \  }}	}
}}|J d       |J d       ||	|
||fS )NTz#new_running_mean should not be Nonez"new_running_var should not be Noner  )r   r   r  r  r  r   r  r  r   r  r  r  r  s                r-   #_native_batch_norm_legit_functionalr  w  sl    " 	!vt\;(CQU	 'N)NN'&L(LL&9i)9?JJr,   c                     |J t        j                  |       |k  j                  t         j                        }|j	                  |       | z  d|z  z  }||fS )Nr   rc   )re   r)  r8   uint8r
  )r   rE  r   r  r+  s        r-   _fused_dropout_decompositionr    s[     OOE"Q&***=D
,,u

%q
1C;r,   c                 d    t        | t        j                  j                        r| j                  S y r5   )r7   re   _subclasses
FakeTensorfake_device)rV  s    r-   device_hintr    s'    &%++667!!!r,   )r   layoutr  
pin_memorynon_blockingrL  r  r  r  rL  c                   |r|t         j                  k(  sJ d       |rJ d       |||| j                         S d}t        |       }|b|| j                  k7  rS|1|j
                  dk(  r"t         j                  j                  | |      } d}t         j                  j                  | |      } |$|s"t         j                  j                  | |      } d}|t        j                  | |      S | S )NTODOFr  TrK  )	re   stridedr  r  r  r  _primsconvert_element_type
device_put)	r9   r   r  r  r  r  rL  dtype_convertedcommon_devices	            r-   _to_copyr#    s     5==08&80!6!>~%-M,AwwyONMf0!511!U;A"OLL##Av.LL--a7 {{1M::Hr,   c                 ,    t         j                  |       S r5   )r   alias)r9   s    r-   nop_decompositionr&    s     ::a=r,   out3exponential_average_factorepsilonc           
         t         j                  | |||||||      \  }}	}
|r%||	|
| j                  dt        j                        fS ||j                  d      |j                  d      | j                  dt        j                        fS )Nr  r   )r   r  ra  re   r  )r   r   r  r  r  r   r(  r)  r  r  r  s              r-   cudnn_batch_normr+    s     $$"	GAq! 1aU[[ABB	EKK0	 r,   c                     t        |      D ]A  \  }}|dk(  s|| j                  k  r| j                  |   ||   k(  r1| j                  |      } C | S rX   )r  rn  r  rQ   )r9   broadcast_maskr  r  s       r-   _broadcast_batch_norm_backwardr.    sU    / "
d19dQVVmPTAU0UD!A" Hr,   c
                   & |j                   }
||j                   }n|
}t        j                  |j                         &&fd| ||||||fD        \  }}}}}}}|j                  }|j	                         }|dk\  sJ d       d}t        t        |            ||   z  }|}|}|r||"J ||J |}t        j                  ||z         }dg|z  }||   ||<   g }t        |      D ]  }||k7  s	|j                  |        t        ||      }d|z  }t        j                  ||      }t        j                  |||z
  z  |      }t        ||z  |      }t        t        j                  ||z  ||z        |      } |t        ||      dz  }!nt        ||z  |      }!|r||z
  | z  }"||"z
  |z
  |!z  }#n||!z  }#|	d   r||z  }$nd }$|	d   r|}%nd }%|#j                  |
      t        |$|      t        |%|      fS )Nc              3   H   K   | ]  }||j                        n|  y wr5   r  r  s     r-   r  z-native_batch_norm_backward.<locals>.<genexpr>  s,      	 $%=a7	s   "r$   z$rank of the input must be at least 2r#   rc   )r   rA   r  r  rK   r  rf  re   r  rP   rS  r.  r   r   r8   r  )'r  r   r   r  r  r  r  r   r  r  r  weight_dtyper  r  r  running_mean_castrunning_var_castsave_mean_castsave_invstd_castr  
input_rankr  num_featuresr   r  r-  reduction_axesrM  r   grad_output_sumdot_p	grad_mean
proj_scale
grad_scaleprojrT   rJ  	grad_biasr:   s'                                         @r-   native_batch_norm_backwardr@    ss    ++K||"33EKK@	 
	 ++KJ?BBB?D[)*[->>LDF).>.JJJ ,1A1MMM -34!"j 0N&t,N4 "N: %9!!!$% *$?DDii~>OIImzD'89>JE./E~VI/		%$,QW0XZhiJ3FNKcQ
3[ .

 T!Z/$t+y8JF
"Z/
1~fn1~#		 	k"K.I|, r,   c
                    t        | |||||||||	
      }|
||f}t        |      D ]2  \  }}|	t        ||   |j                         t	        |||   d       4 |S r  )r@  r  r   r  r   )r  r   r   r  r  r  r  r   r  r  r$  r%  r  r  rT   rM  rF   s                    r-   native_batch_norm_backward_outrB  ]  s    " (F d#J&! Q1=jmQWW5Q
14PQ
 r,   save_varreserveSpacec	                 B    t         j                  || |||||d|g d
      S )NT)TTT)r   r@  )	r   ri   r   r  r  r  rC  r)  rD  s	            r-   cudnn_batch_norm_backwardrF    s5     ** r,   c                    | j                   | j                  t              t        j                  dv fd       | j                  dd  D ]  }t        j                  |dk7  fd         d   |d   z  dk(  rxd   |d   z  dk(  rjt        d t        dd  |      D              }t        d t        dd  ||      D              }t        j                  j                  j                  | ||      S d	 d
 fd} |d   |d         \  }}}}	 |d   |d         \  }
}}}| dt        |d      |
f   }|	s|st        j                  |d      S d } |||||	d      \  }} |||||d      \  }}d }t        t        |j                  d         t        |j                  d               D ]!  \  }}||d|d d |f   }||d|d d |f   z   }# |||z  z  S )Nr  c                      d  S )Nz9adaptive_avg_pool2d(): Expected 3D or 4D tensor, but got r+   r\  s   r-   r~   z%adaptive_avg_pool2d.<locals>.<lambda>  s    KD6R r,   rM  r   c                  "    dt                dS )Nzjadaptive_avg_pool2d(): Expected input to have non-zero size for non-batch dimensions, but input has shape r  r  r  s   r-   r~   z%adaptive_avg_pool2d.<locals>.<lambda>  s     99>uaI r,   rO   c              3   ,   K   | ]  \  }}||z    y wr5   r+   )r  rM  r  s      r-   r  z&adaptive_avg_pool2d.<locals>.<genexpr>  s     G$!QqAvG   c              3   :   K   | ]  \  }}}||d z
  |z  z
    yw)r#   Nr+   )r  rM  r  r~  s       r-   r  z&adaptive_avg_pool2d.<locals>.<genexpr>  s'      
 '1aAQ!O
s   c                 8    t        j                  | |z  |d      S )Ntruncrounding_modere   divr  r  r  s      r-   start_indexz(adaptive_avg_pool2d.<locals>.start_index  s    yyQ99r,   c                 J    t        j                  | dz   |z  |z   dz
  |d      S )Nr#   rN  rO  rQ  rS  s      r-   	end_indexz&adaptive_avg_pool2d.<locals>.end_index  s&    yy!a%1q1,awGGr,   c                    t        j                  |t         j                        } |||       }| |z  dz   }| |z  }|dk(  xs ||z  dk(   }|r|dz  }n
|dk(  r|dz  }t        j                  |t         j                        }|j                  d      |z   }|rUt        j                  | dz
  |j
                  |j                        }	t        j                  ||	      } |||       }
|
|z
  }n|}||||fS )Nr  r#   r   rO   r  )re   r  r  rQ   scalar_tensorr   r  minimum)in_sizeout_sizeorangei0	maxlengthin_size_modadaptive	range_maxr  maxvali1r{  r  rV  rT  s               r-   compute_idxz(adaptive_avg_pool2d.<locals>.compute_idx  s   hvU[[I73 x'!+	(#q(GH{,Ba,GHNIANILL6M	ll2* ((!399SZZF --V,C 68W5B"WFFFIx//r,   .r5  )r   rO   r}   c                     t        |t              r| |fS |dk  sJ ||j                  d      k\  }|dk(  rt        |d      }t	        j
                  | |d      } t        ||       }| |fS )Nr   rO   rM  r5  r   )r7   r   rQ   rS   re   rF  )valsr{  ra  r`  rK   r  s         r-   
maybe_maskz'adaptive_avg_pool2d.<locals>.maybe_mask  sw    fg&< 7N7 0 0 44Dby(q1$$T45D&vt4F<r,   )r`  rK   r   )r  r  rs  re   r   r  r  nnr  
avg_pool2drS   r   r   rP   )r   r  r  rg  kernelrd  idxhlength_hrange_max_h
adaptive_hidxwlength_wrange_max_w
adaptive_wrf  rg  retrM  jr  rV  rn  r  rT  s                      @@@@@r-   adaptive_avg_pool2dru    s.   
 \\FKKEu:D	LLR [[ 
FI	

 Ry;r?"a'E"IB,G1,LG#eBCj+*FGG 
+.uRSz;+O
 
 xx""--eVVDD:H0@ /:%)[QS_.U+D(K.9%)[QS_.U+D(K'a0$67Djzz$H--   hjbND(  hjbND(
 Cdjjn-uTZZ^/DE +1;sAq!|$CS!Q\**C	+
 (X%&&r,   )rj   rV  c                $    t        | |||d|      S )NTinplacerj   
_index_addr9   rK   ru  rV  rj   s        r-   
index_add_r|  	  s     aeVTGGr,   c                $    t        | |||d|      S )NFrw  ry  r{  s        r-   	index_addr~  	  s     aeVU%HHr,   rx  c                8   t        j                  | j                        t        j                  j                  dk  fd       j                  dk(  rj                  d      nd|j                  dkD  r|j                        ndt        j                  k(  fd       dk7  rft        j                  | j                        t        j                  t        k(  xs t        j                  t                    fd       |z  }| j                  dk(  }|r| j                  d      n| }dz  fz   }|rt        j                  nt        j                  }	 |	|||d      }
|r| S |r|
j                  d      S |
j!                         S )	Nr#   c                  $    d j                    dS Nz(Index should have dimension 1 or 0 (got r0  r\  ru  s   r-   r~   z_index_add.<locals>.<lambda>,	      :5::,aH r,   r   c                      d d d S )NzNumber of indices (z') should be equal to tensor.size(dim) (z), for dim=r+   )rK   
index_sizerW  s   r-   r~   z_index_add.<locals>.<lambda>2	  s!    %j\1XYdXeeqmplrs r,   c                  (    dt                d dS )Nzalpha argument of type z cannot be safely cast to type !)r  )rj   python_types   r-   r~   z_index_add.<locals>.<lambda>9	  s    -d5k]:YZeYffgh r,   r5   Tr  )rA   canonicalize_dimsrn  re   r   r   dtype_to_typer   r'  is_weakly_lesser_typer  rQ   r   
index_put_	index_putr  r  )r9   rK   ru  rV  rx  rj   zero_dimrH  r  r  r   r  r  rW  s    ``  `     @@@r-   rz  rz   	  sM    
!
!!&&#
.C	LL

aH #(**/AqJ&,kkAo&++c"1K	LLz!s z))!''24 E**4;Dh	

 %vv{H#QB
C-5(
"C#*I
BV
5C!)s{{1~?s~~/??r,   c           
         t        j                  t        |       dkD  d        t        |       }| d   j                         }|dd  }t	        d | D              }|r||f}n||f}||z   }| d   j                  ||      }dt        |      z  }	t        |      D ]j  }
| |
   }t        j                  ||	d||j                  d      z
  fz   |      }|rt        j                  ||d|
      }Rt        j                  ||d|
      }l |S )Nr   c                       y)Nz#received an empty list of sequencesr+   r+   r,   r-   r~   zpad_sequence.<locals>.<lambda>K	  r  r,   r#   c              3   >   K   | ]  }|j                  d         ywr  r   )r  r9   s     r-   r  zpad_sequence.<locals>.<genexpr>O	  s     /!&&)/s   )r   r   rK   ru  )
re   r   rs  r   r   r:  rP   r   rQ  rw  )	sequencesbatch_firstpadding_valuesequences_sizemax_sizetrailing_dimsmax_lenout_dimsr   dim_paddingsrM  currseqrows                r-   pad_sequencer  H	  s#    
LLY!#%RS^N|  "HQRLM/Y//G"G,^,-'H
A,

-
8CC..L>" @A,""\Q',,q/(A$BBM
 %%c3AQ%?C%%c3AQ%?C@ Jr,   c                 "    t        | |||d      S )NTrx  _index_copyr9   rK   ru  rV  s       r-   index_copy_r  c	  s    q#ufd;;r,   c                 "    t        | |||d      S )NFr  r  r  s       r-   
index_copyr  h	  s     q#ufe<<r,   c                   t        j                  | j                  |      }t        j                  j                  dk  fd       | j                  dk(  }|r| j                  d      n| }j                  dk(  rj                  d      nd|z  fz   }|rt        j                  nt        j                  } ||||      }	|r| S |r|	j                  d      S |	j                         S )Nr#   c                  $    d j                    dS r  r\  r  s   r-   r~   z_index_copy.<locals>.<lambda>t	  r  r,   r   r5   )rA   r  rn  re   r   rQ   r   r  r  r  r  )
r9   rK   ru  rV  rx  r  rH  r  r  r   s
     `       r-   r  r  n	  s     
!
!!&&#
.C	LL

aH
 vv{H#QB"'**/EOOAuE
C-5(
"C#*I
BV
$C!)s{{1~?s~~/??r,   c                    t        j                  | j                  d      |       }t        j                  t        j                  |              }| j
                  r| j                  d      }n|}|t        j                  |      z
  |fS )Nr+   r  )re   rY  ra  rd   r   r  r9  )rx   r   rg   r   s       r-   log_sigmoid_forwardr  	  sh     --r*D
1C		599T?"#A||%Q''r,   lowhighc                     t        j                  | j                  t        |      t        |      | j                  | j
                  |      S )N)r  r  r   r  r   )prims_uniform_helperr  r   r   r  )r9   r  r  r   s       r-   r   r   	  s=       	cNt_ggxx r,   c                 <    | j                  t        | |||            S r5   )r   r   )rx   r  r  r   s       r-   uniform_r  	  s    ::gdCy9::r,   c                 "   t        |       dz
  }|>t        j                  |d u d        t        j                  t        |      |k(  d        |S |t        j                  |d u d        t        j                  t        |      |k(  d        g }t        |      D ]Z  \  }}t	        |      |k(  r$|j                  | |dz      t	        |      z         8|j                  t        | |dz      |z               \ |S t        j                  dd        y )Nr$   c                       yNz9Must specify exactly one of output_size and scale_factorsr+   r+   r,   r-   r~   z.upsample_compute_output_size.<locals>.<lambda>	  r  r,   c                       yN r+   r+   r,   r-   r~   z.upsample_compute_output_size.<locals>.<lambda>	  r  r,   c                       yr  r+   r+   r,   r-   r~   z.upsample_compute_output_size.<locals>.<lambda>	  r  r,   c                       yr  r+   r+   r,   r-   r~   z.upsample_compute_output_size.<locals>.<lambda>	  r  r,   Fc                       yr  r+   r+   r,   r-   r~   z.upsample_compute_output_size.<locals>.<lambda>	  r  r,   )rs  re   r   r  r  rS  r   )r  r  scale_factorsspatial_dimensionsrM  r~  s         r-   upsample_compute_output_sizer  	  s   Z1,T!O	
 	S%);;ZH 4O	
 	S'+==zJm, 	CDAq1v{"":a!e#4s1v#=>""7:a!e+<q+@#AB		C
 	LLRr,   c                     | y | |   S r5   r+   )scalesr  s     r-   get_scale_valuer  	  s    ~#;r,   c                     t        | j                         ||      }t        |d      }t        j                  j                  | ||      S rp   )r  r   r  r   upsample_nearest1dr  r   r  r  osizerk   s        r-   upsample_nearest1d_vecr  	  s?     ){MREM1-E""**5%??r,   c                     t        | j                         ||      }t        |d      }t        j                  j                  | ||      S rp   )r  r   r  r   _upsample_nearest_exact1dr  r  s        r-   _upsample_nearest_exact1d_vecr  	  s?     ){MREM1-E))11%FFr,   c                     t        | j                         ||      }t        |d      }t        |d      }t        j                  j                  | |||      S Nr   r#   )r  r   r  r   upsample_nearest2dr  r   r  r  r  scale_hscale_ws         r-   upsample_nearest2d_vecr  	  sM     ){MREmQ/GmQ/G""**5%'JJr,   c                     t        | j                         ||      }t        |d      }t        |d      }t        j                  j                  | |||      S r  )r  r   r  r   _upsample_nearest_exact2dr  r  s         r-   _upsample_nearest_exact2d_vecr  	  sM     ){MREmQ/GmQ/G))11%QQr,   c                     t        | j                         ||      }t        |d      }t        |d      }t        |d      }t        j                  j                  | ||||      S r   )r  r   r  r   upsample_nearest3dr  r   r  r  r  scale_dr  r  s          r-   upsample_nearest3d_vecr  	  s[     ){MREmQ/GmQ/GmQ/G""**5%'7SSr,   c                     t        | j                         ||      }t        |d      }t        |d      }t        |d      }t        j                  j                  | ||||      S r   )r  r   r  r   _upsample_nearest_exact3dr  r  s          r-   _upsample_nearest_exact3d_vecr  
  s`     ){MREmQ/GmQ/GmQ/G))11ugw r,   c                    g }t        |      }|rdnd}t        |      D ]  }||   }| j                  | |z      }	||   |	|	||   z  z  n|	|z  }
t        j                  |t        j
                  | j                        }||z   |
z  j                  t        j                        }t        |dz
  |z
        D ]  }|j                  d      } |j                  |        t        |      S )Nr   r   r  r#   rO   )rs  rP   r  re   r  r   r  r8   r  rQ   rS  r  )r   r  r  exactr<  num_spatial_dimsry  r  r  isizerk   output_indicesinput_indicesrR   s                 r-   !_compute_upsample_nearest_indicesr  
  s     G;'SsF#$ & A--12/5ay/D*+%RW-e5==V(61U:>>u{{K'!+a/0 	8A)33B7M	8}%+&, >r,   r  c                 T    t        | ||f      \  }t        j                  | d d |f      S r5   r  r   _unsafe_indexr   r  r  	l_indicess       r-   r  r  /
  s0     5UK&SLYedD)%<==r,   c                 X    t        | ||fd      \  }t        j                  | d d |f      S NT)r  r  r  s       r-   r  r  ;
  s7     5{VITLY edD)%<==r,   c                    t         j                  | d d ||f      }t        j                  |       }| j                  \  }}}}| j
                  j                  dk(  r|dk  rt        j                  }|j                  |      }|S )Ncudar5  rK  )
r   r  rA   r  r  r  r  re   rP  r  )r   	h_indices	w_indicesr  rL  rR   
n_channelss          r-   _upsample_nearest2d_commonr  I
  s~    dIy'IJF //6M  ++Az1a||F"zA~//];FMr,   scales_hscales_wc                 @    t        | |||f      \  }}t        | ||      S r5   r  r  r   r  r  r  r  r  s         r-   r  r  X
  s0     ={Xx0Iy &eY	BBr,   c                 D    t        | |||fd      \  }}t        | ||      S r  r  r  s         r-   r  r  g
  s2     ={Xx0Iy &eY	BBr,   scales_dc                 d    t        | ||||f      \  }}}t        j                  | d d |||f      }|S r5   r  	r   r  r  r  r  	d_indicesr  r  r  s	            r-   r  r  v
  sH     'H{Xx:'#Iy) dIy)'TUFMr,   c                 h    t        | ||||fd      \  }}}t        j                  | d d |||f      }|S r  r  r  s	            r-   r  r  
  sJ     'H{Xx:$'#Iy) dIy)'TUFMr,   c           	          |r|rd}n|rd}n|rd}nd}t        |       |z  dk(  sJ t        |              t        dt        |       |      D cg c]  }t        | |||z           c}S c c}w )Nr  r5  r   r$   r   )rs  rP   r  )params
has_biaseshas_projections
group_sizerM  s        r-   gather_paramsr   
  s{    o
	
	

v;#q(5#f+5(38CK3T./fQZ()  s   A(c                 ~    |r'| d|z     |d|z     }}| d|z  dz      |d|z  dz      }}n| |   ||   }}d\  }}||||fS )Nr$   r#   NNr+   )r  hiddensrM  bidirectional
cur_params
cur_hiddenbidir_paramsbidir_hiddens           r-   params_hiddensr	  
  sk    !'AAJ
%+AEAI%6A	8Jl!'GAJJ
%/"lz<==r,   c                     ||kD  sJ |j                  | j                  d|||z
               | j                  dd|      S rp   )rS  r!  )r  last_batch_size
batch_sizer  s       r-   update_hidden_for_packedr  
  sE    Z'''NN:$$Q
Oj4PQRQ:..r,   c           	      t    ||k(  r| S ||k  sJ t        j                  | |j                  d|||z
        f      S rp   )re   concatr!  )r  r  r  
inp_hiddens       r-    update_hidden_for_packed_reverser  
  sP     *$Z'''<<a*2NO	
 r,   c           	      X   |d   }|d   }|r|d   nd }	|r|d   nd }
g }g }|r|d   n|d   }|j                  dd|      }t        j                  | t        |            }|r|d d d   }|D ]V  } | j                  d   }||k(  rn|rt        ||||      }nt        ||||      } || |||	||
      }|}|j                  |       X |r|j                          n!|j                  |       |j                          t        j                  |d      }|st        j                  |d      n|}||fS )Nr   r#   r$   r   rO   )
r!  re   r  rf  r  r  r  rS  reverser"  )inphiddenr  r  	hidden_fnbatch_sizesr  	ih_weight	hh_weightih_biashh_biasstep_outputr  r  r  	split_inprM  r   
hidden_outs                      r-   one_layer_rnn_datar  
  sR    q	Iq	I%fQi4G%fQi4GK"$G)0k"ok!nOq!_5JCk!23IddO	 'IIaLa9OQJ 2OQJ sJ	7IwW
:&#'& z"
))K
#C.57A&:J
?r,   c                       fd}|S )Nc                 D     t        j                  |||      | z         S r5   r   linearrM  r  r  r  r  r  nonlinearitys         r-   rG   zrnn_cell.<locals>.inner
  s     AHHZGDqHIIr,   r+   r%  rG   s   ` r-   rnn_cellr'  
  s    J Lr,   c                       fd}|S )Nc                 r    t        j                  | ||      }  t        j                  |||      | z         S r5   r"  r$  s         r-   rG   zrnn_cell_data.<locals>.inner
  s2    HHQ	7+AHHZGDqHIIr,   r+   r&  s   ` r-   rnn_cell_datar*  
  s    J Lr,   c           	      x   |d   }|d   }|r|d   nd }|r|d   nd }	t        j                  | ||      }
|r|
j                  d      n|
}
|j                  d      }g }|
D ]   } |||||||	      }|j	                  |       " |r|j                          t        j                  |d      }||j                  d      fS )Nr   r#   r$   r   )	r   r#  fliprQ   rS  r  re   r"  r  )r  r  r  r  r  r  r  r  r  r  precomputed_inputr  r  rM  r   s                  r-   one_layer_rnnr.    s    q	Iq	I%fQi4G%fQi4Gi95<)..q1BS!!!$JK 'q*i)WU
:&' 
))K
#C
""1%%%r,   c                    |d   }|d   }|r|d   }|d   }nFt        j                  |j                               }t        j                  |j                               }|d   j                  d      }	|d   j                  d      }
g }d}|	j                  d      }d}d}d}d}| j	                         } |	j	                         }	|
j	                         }
t         j
                  j                  j                  j                  | |||||	|
|||||||||      }|d   |d   |d   }}}||j                  d      |j                  d      ffS )Nr   r#   r$   r   F)
re   zerosr   rQ   r  r  r   mkldnn_rnn_layerr  r  )r  r  r  r  r  w0w1w2w3hxcxr  modehidden_size
num_layersr  r  r   outputsrV   hycys                         r-   mkldnn_one_layer_lstmr>    sS   	B	BAYAY[[#[[#			Q	B			Q	BKD''!*KJ MKE ..
C	B	Biinn--55





!G$ 
GAJ
2rArzz!}bjjm,,,r,   c
                    |r| j                  dd      n| } g }
t        |      D ]  }t        ||||      \  }}}}|r
||dz
  k  r|nd} |	| |||      \  }}|
j                  |       |r! |	| |||d      \  }}|
j                  |       |r*t	        j
                  |g|j                         dz
        } n|} |dk7  s|s||dz
  k  st	        j                  | |d      }  |r| j                  dd      n| } | |
fS )Nr   r#   r   T)r  )r   )	transposerP   r	  rS  re   r"  rK   r#  )r   r  r  r  r:  r#  r   r  r  layer_fnfinal_hiddensrM  r  r  r  r  fwd_inp
fwd_hiddenbwd_inp
bwd_hiddens                       r-   _rnn_helperrG  N  s"    &1EOOAq!eEM: >=KFA}>
:
Jl $
QU(:'&uj*jQZ("*|\:t#GZ   ,IIw0'++-!2CDEEa<Ea*q.&8MM%=E)>, &1EOOAq!eE-r,   c	                     |j                  d      }	t        ||d      }t        | |	|||||||t        t        t        t        j                              
      \  }
}|
t        j                  |d      fS Nr   Fr  )	unbindr   rG  r   r.  r'  re   r   stackr   r6  r  r  r:  r#  r   r  r  r  r   rB  s               r-   rnn_tanh_inputrN  w  t     YYq\F6:u5F$%**)=>C M1---r,   c	                     |j                  d      }	t        ||d      }t        | |	|||||||t        t        t        t        j                              
      \  }
}|
t        j                  |d      fS rI  )	rK  r   rG  r   r.  r'  re   r  rL  rM  s               r-   rnn_relu_inputrQ    rO  r,   c	                     |j                  d      }	t        ||d      }t        | |	||||||dt        t        |t        t        j                              
      \  }
}|
t        j                  |d      fS Nr   Fr  r  )	rK  r   rG  r   r  r*  re   r  rL  datar  r6  r  r  r:  r#  r   r  r  r   rB  s               r-   rnn_relu_datarW    {     YYq\F6:u5F$##EJJ/	
C  M1---r,   c	                     |j                  d      }	t        ||d      }t        | |	||||||dt        t        |t        t        j                              
      \  }
}|
t        j                  |d      fS rS  )	rK  r   rG  r   r  r*  re   r   rL  rU  s               r-   rnn_tanh_datarZ    rX  r,   c                 l   t        j                  |||      | z   }|j                  d|      }|d   j                         }	|d   j                         }
|d   j	                         }|d   j                         }|
|z  |	|z  z   }||j	                         z  }||nt        j                  ||d       }||fS )Nr5  r   r#   r$   r   r   r#  chunkr   r   )r  r6  r7  r  r  	hr_weight	chunk_dimgateschunked_gatesin_gateforget_gate	cell_gateout_gater=  r<  s                  r-   	lstm_cellrf    s    HHRG,s2EKK9-MA&&(G"**,Ka %%'IQ'')H	r	Wy0	1B	BGGI	B ahhr9d&CBr6Mr,   c           
      (   |d   }|d   }|r|d   nd }|r|d   nd }t        |      dk(  r|d   nt        |      dk(  r|d   nd }	|d   j                  d      }
|d   j                  d      }t        j                  | ||      }|r|j	                  d      n|}g }|D ](  } t        | |
||||	d      \  }
}|j                  |
       * |r|j                          t        j                  |d      }||
j                  d      |j                  d      ffS )Nr   r#   r$   r   r  r5  r_  )rs  rQ   r   r#  r,  rf  rS  r  re   r"  r  )r  r  r  r  r  r  r  r  r  r^  r6  r7  r-  r  r   s                  r-   one_layer_lstmri  	  s*   q	Iq	I%fQi4G%fQi4G[A%q	Fq8H6!9d  
		Q	B			Q	Bi95<)..q1BSK  3B	7IQRSB2 
))K
#CA

1...r,   c           
         |d   }|d   }|r|d   nd }|r|d   nd }	t        |      dk(  r|d   nt        |      dk(  r|d   nd }
g }g }|r|d   n|d   }t        j                  | t        |            }|r|d d d   }|d   }|d   }|j	                  dd|      |j	                  dd|      }}|D ]  } | j
                  d   }t        j                  | ||      } ||k  ra|j                  |j	                  d|||z
        |j	                  d|||z
        f       |j	                  dd|      |j	                  dd|      }}||kD  rXt        j                  ||j	                  d|||z
        fd      }t        j                  ||j	                  d|||z
        fd      }t        | ||||	|
d      \  }}|}|j                  |        |r|j                          ||f}nZ|j                  ||f       |j                          t        | \  }}t        j                  |d      t        j                  |d      f}t        j                  |d      }||fS )	Nr   r#   r$   r   r  r5  rO   rh  )rs  re   r  rf  r!  r  r   r#  rS  r  rf  r  r  r"  )r  r  r  r  r  r  r  r  r  r  r^  r  r  r  r  orig_hxorig_cxr6  r7  rM  r  hidden0hidden1r   s                           r-   one_layer_lstm_dataro  $  sx   q	Iq	I%fQi4G%fQi4G[A%q	Fq8H6!9d  KG)0k"ok!nOCk!23IddO	QiGQiG^^Aq/2GNN	1o5B  IIaLhhsIw/ NNIIaOa$78IIaOa$78 YYq!Q'1a);B W^^AO8KLMqB W^^AO8KLMqB 3B	7IQRSB236 "X
Bx =YYw*EIIgq,AA

))K
#C
?r,   c                 4    d } || ||      rt         S t        S )a*  Check whether we could use decompose lstm with mkldnn_rnn_layer.
    All the below conditions need to be met:
        * ``torch._C._get_mkldnn_enabled()`` returns ``True``.
        * All the input args are on CPU.
        * The dtypes of args are either torch.float or torch.bfloat16.
        * Inference.
        * ``has_projections`` returns ``False``.

    Args:
        * input: the input sequence to LSTM
        * hx: a tuple of the input hidden state and cell state ``(h_0, c_0)`` to LSTM
        * params: the weight and bias tensors of LSTM
    c                 N   t         j                  j                         sy| gt        |      z   t        t	        j
                  |            z   }|D ch c]  }|j                   }}t        |      dk7  ry|j                         }|t        j                  d      k7  ry|D ch c]  }|j                   }}|D ]&  }|t         j                  t         j                  fvs& y | j                  ry|d   j                  d      |d   j                  d      k7  }	|	ryyc c}w c c}w )NFr#   r  r   r$   T)re   _C_get_mkldnn_enabledrf  r   from_iterabler  rs  popr   r  bfloat16requires_gradr   )
r   r6  r  rN  tdevicesr  dtypesr   r  s
             r-   
use_mkldnnz2select_one_layer_lstm_function.<locals>.use_mkldnnr  s    xx++-'DH$tE,?,?,G'HH%,-188--w<1U\\%((#*+a!''++ 	EU[[%..99	 Q%**Q-2a5::a=8) . ,s   D#D")r>  ri  )r   r6  r  r{  s       r-   select_one_layer_lstm_functionr|  c  s!    : %V$$$r,   c	                    t        |      dk(  sJ d       t        |||d   j                  d      |d   j                  d      k7        }t        t	        |d   |d               }	t        | ||      }
t        | |	||||||||

      \  }}t        t	        |       }|t        j                  |d   d      t        j                  |d   d      fS )Nr$   lstm expects two hidden statesr   r#   )	rs  r   r   rf  r  r|  rG  re   rL  )r   r6  r  r  r:  r#  r   r  r  r  rA  r   rB  s                r-   	lstm_implr    s     r7a<999<6:r!uzz!}1

1/MNF#beRU#$F-eR@H$C m,-MM!,a0%++mA>NPQ2RRRr,   c	                    t        |      dk(  sJ d       t        |||d   j                  d      |d   j                  d      k7        }t        t	        |d   |d               }	t        | |	||||||dt        t        |      
      \  }
}t        t	        |       }|
t        j                  |d   d      t        j                  |d   d      fS )Nr$   r~  r   r#   F)r  )
rs  r   r   rf  r  rG  r   ro  re   rL  rU  s               r-   lstm_data_implr    s     r7a<999<6:r!uzz!}1

1/MNF#beRU#$F$#=C m,-MM!,a0%++mA>NPQ2RRRr,   c                 &   | j                  dd      }t        j                  |||      j                  dd      }|d   |d   z   j                         }|d   |d   z   j                         }	|d   |d   |z  z   j	                         }
||
z
  |	z  |
z   S )Nr   r#   r$   r   )r]  r   r#  r   r   r  r  r  r  r  r  chunked_igateschunked_hgates
reset_gate
input_gatenew_gates              r-   gru_cellr    s    YYq!_NXXj)W=CCAqIN #nQ&77@@BJ #nQ&77@@BJq!^A%6%CDJJLH!Z/(::r,   c                 P   t        j                  | ||      j                  dd      }t        j                  |||      j                  dd      }|d   |d   z   j                         }|d   |d   z   j                         }	|d   |d   |z  z   j	                         }
||
z
  |	z  |
z   S )Nr   r#   r   r$   r\  r  s              r-   gru_cell_datar    s    XXc9g6<<QBNXXj)W=CCAqIN #nQ&77@@BJ #nQ&77@@BJq!^A%6%CDJJLH!Z/(::r,   c	                     t        ||d      }t        | |j                  d      ||||||dt        t        |t
              
      \  }	}
|	t        j                  |
d      fS )NFr   rT  )r   rG  rK  r   r  r  re   rL  )rV  r  r6  r  r  r:  r#  r   r  r   rB  s              r-   gru_impl_datar    si     6:u5F$
		!"}UC M1---r,   c	                     t        ||d      }t        | |j                  d      |||||||t        t        t
              
      \  }	}
|	t        j                  |
d      fS )NFr   rJ  )r   rG  rK  r   r.  r  re   rL  )r   r6  r  r  r:  r#  r   r  r  r   rB  s              r-   gru_implr    sf     6:u5F$
		!2C M1---r,   c                     t        | j                         ||      }t        |d      }t        |d      }t        j                  j
                  j                  | ||||      S r  )r  r   r  re   r  r   _upsample_bilinear2d_aar   r  align_cornersr  r  r  r  s          r-   upsample_bilinear2d_aa_vecr  &  sV     ){MREmQ/GmQ/G99>>11umWg r,   c                     t        | j                         ||      }t        |d      }t        |d      }t        j                  j
                  j                  | ||||      S r  )r  r   r  re   r  r   _upsample_bicubic2d_aar  s          r-   upsample_bicubic2d_aa_vecr  2  sV     ){MREmQ/GmQ/G99>>00umWg r,   c                 z    t        | j                         ||      }|r|nd gt        |      z  }t        | |||      S r5   )r  r   rs  _upsample_linear)r   r  r  r  r  r  s         r-   _upsample_linear_vecr  >  s=     ){MRE+]$#e*1DFE5-@@r,   r  c                      t        | |||g      S r5   r  )r   r  r  r  s       r-   upsample_linear1dr  L  s     E;zJJr,   c                 "    t        | ||||g      S r5   r  )r   r  r  r  r  s        r-   upsample_bilinear2dr  W  s     E;(?STTr,   c                 $    t        | |||||g      S r5   r  )r   r  r  r  r  r  s         r-   upsample_trilinear3dr  f  s!     {MHh+I r,   c                 L    |r|dkD  r| dz
  |dz
  z  S dS |
|dkD  rd|z  S | |z  S )Nr#   rc   r   r+   rZ  r[  r  rk   s       r-   _compute_scaler  w  sB    5=\#(S.1HqH#/EAIsU{U7XCUUr,   c                 &    |r| |z  S | |dz   z  dz
  S Nr   r+   rk   	dst_indexr  s      r-   _compute_source_indexr  ~  s$    y  	C(3..r,   c                      j                   d d \  }} j                   dd  }t        |      }t        j                   t        j                  j
                        \  } fd}	t        t        |||            D 
cg c]  \  }
\  }}} |	||||dz
  |
z
         }}}}
}t        t        |       \  }}}g }t        ddgg|z   D ]c  }d d gt        |      D cg c]  }||   dk(  r||   n||    c}z   }t        j                   |      }t        |      }|j                  |       e t        t        |            D ]p  }
||
   ||
   z
  j!                  dd      j#                        }t        |d d d   |dd d         D cg c]!  \  }}|t%        j&                  ||z
  |      z   # }}}r t        |      dk(  sJ |d   }t        j(                         } j*                  j,                  dk(  r|d	k  rt$        j.                  }t1        |t$        j2                        sJ |j5                  |
      } j7                         s|j9                         }|S c c}}}}
w c c}w c c}}w )Nr$   r/  c                 |   t        | |	|      }t        j                  |j                        j	                  
      }t        ||	      j                  d      } |j                  |j                  d   gdg|z   }|j	                  t        j                        }|dz   j                  | dz
        }|||fS )Nr  r   r   r   r   r#   r   )
r  re   r  r  r8   r  r   r  r  r  )inp_sizer[  r  nsqueezescale_factorrM  x_f32r9   xp1r  r   r   s            r-   
get_valuesz$_upsample_linear.<locals>.get_values  s    %h-P LL%,,7:::G%lA}EKKPSKTekk!n@sh/?@HHU[[!1umm1m-a}r,   r#   r   r   rc   r     rK  )r  rs  rA   rB   r2  INT_TO_FLOATr  r  rf  r   rP   r   r  r   rS  reversedr   r8   re   r   r  r  r  rP  r7   r   r  r(  round)r   r  r  r  n_batchr  	inp_sizesn_dimsrR   r  rM  r  r[  valuesxs_f32xsxp1svsr  kr  vxscalev1v2r  rL  r   s   ` `                        @r-   r  r    s     ++bq/GZABI^F''!AANNHAu
 09	;/0
 +A+(F 	8XvvzA~>F  CL)FB	B1vh'( TluV}U!qtqybed1g=UUuc*#Au-
		!	 eFm$ 
)be#**3477> b1gr!$Q$x0
 B 27F++
 

 r7a<<UF //6M ||F"zB//fell+++];F""$MQ V
s   I"
 I*&I/r  r  c                 4    | j                   |j                   k(  S r5   r  )r  r  s     r-   is_same_sizer    s    77aggr,   c                 .    t         j                  | |      S r5   )r   rT  )r9   r  rC   s      r-   _reshape_aliasr    s     99Qr,   c                 .    t         j                  | |      S r5   )r   ru  )r9   r<  s     r-   _indexr    s    ::a!!r,   c                    | j                         }d}|dk  rd}|6|dkD  r*dg|z  }|j                  d   ||<   |j                  |      }n|}| |z  } t        j                  ||k7  |d      }	|	j                  |      }
t        j                  | ||
      j                  |       }t        j                  ||k7  |d      }|t        j                  j                  k(  r|dkD  r| j                  dd      }||fS |lj                  | j                        }t        j                  |||
      j                  |      }t        j                  ||k7  |d      }|j                         }n"||k7  j                         j                  |       }|t        j                  j                  k(  r|j                         }||fS |t        j                   j                  k(  r|j                         |z  }||fS )Nr#   r$   r   r+   r   )rK   r  rT  re   rf   rQ   gatherr  r"   r(   ry   r:  expandr   r8   r*   r)   )rx   r   r   r   r  r  r  r  wr  safe_target_r  r  wsums                 r-   _nll_loss_forwardr    s    XXZFKzA:E "(aE+E"AAax++f4fa@K((5L ll4l;CCKPPF[[</;FINN(((VaZ}}R-|##HHTZZ ||A{L9AA+N{{6\14;xxz,.33588>IMM''' < 
inn**	*,<r,   c                     | j                         dkD  r| j                         dk  sJ d       |j                         dk  sJ d       | j                         dk(  xr |j                         dk(  }|sA| j                  d   |j                  d   k(  s"J d| j                   d|j                   d       | j                  d	   }|=|j                         dk(  r|j                         |k(  sJ d
| d|j                          t        | ||||      S )Nr   r$   r,  r#   r-  r.  r/  r0  rO   z/weight tensor should be defined either for all z7 classes or no classes but got weight tensor of shape: )rK   r  r   r  )rx   r   r   r   r  r2  	n_classess          r-   nll_loss_forwardr    s1    88:>dhhjAoP/PP-

EDE 88:?8vzz|q'8L

1a(J	#DJJ<z&,,qIJ  

2I>

flln	9J	8Cz  |B  |H  |H  {I  JJ  T669lKKr,   c                      t        | ||||      S r5   )r  )rx   r   r   r   r  s        r-   nll_loss2d_forwardr  0  s     T669lKKr,   Ac                 0    |dz   | z  |dz   z
  | z  | z  dz   S )Nr$   r   r#   r+   r9   r  s     r-   _upsample_cubic_convolution1r  >  s(    UaK1q5!Q&*Q..r,   c                 <    || z  d|z  z
  | z  d|z  z   | z  d|z  z
  S )Nr     r5  r+   r  s     r-   _upsample_cubic_convolution2r  B  s0    UQU]a!a%'1,q1u44r,   rx  c                 t    d}t        | dz   |      t        | |      t        d| z
  |      t        d| z
  |      fS )Ng      rc   r   )r  r  )rx  r  s     r-    _upsample_get_cubic_coefficientsr  F  sE    A$QWa0$Q*$S1Wa0$S1Wa0	 r,   coeffstsc                 P    t        |      }t        d t        | |      D              S )Nc              3   ,   K   | ]  \  }}||z    y wr5   r+   )r  r  r  s      r-   r  z+_upsample_cubic_interp1d.<locals>.<genexpr>R  s     EHRRErK  )r  _sum_tensorsr  )r  r  coeffs2s      r-   _upsample_cubic_interp1dr  P  s$    .r2GEFG0DEEEr,   c                 6    t        t        j                  |       S r5   )r   re   add)r  s    r-   r  r  V  s    %))R  r,   	num_stepsc                     | dk  rt        j                  d||      S |s| dz
  | z  nd}t        j                  | || ||      S )Nr#   r   r  )stepsr  r   )re   rV  linspace)r  r  r   r  r  s        r-   _linspace_from_neg_oner  Z  sI     A~||AfE::-:)a-9	$A>>1"ayuMMr,   thetahr  c                    | j                   }| j                  }t        ||||      j                  d|d      }t        ||||      j                  |dd      }t	        j
                  d||      }t        j                  j                  j                  |ddd      }t        j                  j                  j                  |ddd      }t        j                  j                  j                  |d	dd      }||z   |z   S )
Nr#   )r#   r#   r#   r  )r   r$   constantr   r  r8  ry   r#   r#   )r$   r   	r   r  r  rT  re   rH  rh  r  r  )	r  r  r  r  r   r  grid_xgrid_ygrid_ones	            r-   _make_base_grid_4dr  d  s    KKE\\F $A}eVDII!QPQRF#A}eVDII!QPQRFzz)5@H XX  $$VjPQ$RFXX  $$VjPQ$RFxx""&&xV*TU&VHF?X%%r,   r  c                    | j                   }| j                  }t        ||||      j                  dd|d      }t        ||||      j                  d|dd      }t        ||||      j                  |ddd      }	t	        j
                  d||      }
t        j                  j                  j                  |ddd      }t        j                  j                  j                  |ddd      }t        j                  j                  j                  |	d	dd      }	t        j                  j                  j                  |
d
dd      }
||z   |	z   |
z   S )Nr#   )r#   r#   r#   r#   r  )r   r   r  r   r  )r#   r$   )r$   r#   )r   r   r  )r  r  r  r  r  r   r  r  r  grid_zr  s              r-   _make_base_grid_5dr  u  s5   KKE\\F#A}eVDII!QPQSTUF#A}eVDII!QPQSTUF#A}eVDII!QPQSTUFzz,eFCH XX  $$VjPQ$RFXX  $$VjPQ$RFXX  $$VjPQ$RFxx""&&xV*TU&VHF?V#h..r,   c                     |\  }}}}t        | |||      }|j                  ddd      | j                  j                  d      z  j	                  d      }|j                  |||d      S )Nr  rO   r   r#   rM  r$   )r  rT  rR  rQ   r   )	r  r   r  r  rR   r  r  	base_gridgrids	            r-   _affine_grid_generator_4dr    sg    JAq!Q"5!QmLI NN2q!$uxx'9'9!'<<AA"ED99Q1a  r,   c                     |\  }}}}}t        | ||||      }|j                  ddd      | j                  j                  d      z  j	                  d      }	|	j                  ||||d      S )Nr  rO   r5  r#   rM  r   )r  rT  rR  rQ   r   )
r  r   r  r  rR   r  r  r  r  r  s
             r-   _affine_grid_generator_5dr     sm    MAq!Q"5!QOI NN2q!$uxx'9'9!'<<AA"ED99Q1a##r,   c                     t        j                  t        |      dv d        t        |      dk(  rt        | ||      S t	        | ||      S )N)r5  r  c                       y)NzCaffine_grid_generator needs 4d (spatial) or 5d (volumetric) inputs.r+   r+   r,   r-   r~   z'affine_grid_generator.<locals>.<lambda>  r  r,   r5  r  )re   r   rs  r  r   )r  r   r  s      r-   affine_grid_generatorr    sJ     
LLD	VU 4yA~(MRR(MRRr,   r  interpolation_modepadding_mode_expand_gridc           	          !"#$%&'()*+,- t        j                  dv fd       t        j                  dv fd       dt        dt        dt        ffd-dt        dt        d	t        dt        fd
+dt        dt        dt        f+fd dt        dt        dt        f -fd} j                  \  $%|j                  \  })*}|dk(  sJ r(|j                  d)*|      j                  )*d      }dt        dt        dt        f$%fd&t        j                   j                        j                  ddd      t        j                   j                        j                  ddd      dt        dt        dt        dt        f&)*fddt        dt        dt        f fd"|d   }	|d   }
dk(  r ||	%      } ||
$      }|j                         |j                         c'('dz   (}}'(dz   }}||}}||z
  ||z
  z  }||z
  ||z
  z  }||z
  ||z
  z  }|'z
  |(z
  z  }t        "fd'(|f|||f|||f|||ffD              S dk(  r< ||	%      } ||
$      }|j                         }|j                         } "||d      S  -|	%      } -|
$      }|j                         '|j                         (|'z
  ,|(z
  }s",j                  d      ,|j                  d      }dt        dt        dt        f "$%fd#dt        dt        f#'(,fd!t        !fdt        d       D              }t!        ||      S )!N)r   r#   r$   c                      d  S )NzInvalid interpolation mode r+   )r  s   r-   r~   z"_grid_sampler_2d.<locals>.<lambda>  s    -.@-AB r,   c                      d  S )NzInvalid padding mode r+   )r  s   r-   r~   z"_grid_sampler_2d.<locals>.<lambda>  s    -B<.+Q r,   coordsr   rL   c                 B    r|dz  dz
  n|dz  }|dz  dz
  }| |z  |z   S r  r+   )r
  r   r   ofsr  s       r-   unnormalizez%_grid_sampler_2d.<locals>.unnormalize  s8     %2tczCs
Sj3|c!!r,   	twice_low
twice_highc                 P   ||k(  rt        j                  |       S |dz  }||z
  dz  }| |z
  j                         }t        j                  ||      }||z  j	                         j                  t         j                        }t        j                  |dz  dk(  ||z   ||z   |z
        S )Nr$   r   r#   r   )re   r  r   fmodfloorr8   int8rf   )r
  r  r  
coords_mincoords_spancoords2extraflipss           r-   reflect_coordinatesz-_grid_sampler_2d.<locals>.reflect_coordinates  s    
"##F++]
!I-2J&++-

7K0;&--/222D{{AINEJ.j0H50P
 	
r,   c                     dk(  r| S dk(  rt        j                  | d|dz
        S r | dd|dz
  z        }n | dd|z  dz
        }t        j                  |d|dz
        S )Nr   r#   r$   rO   r   )r
  r   coords_reflectedr  r  r  s      r-   compute_coordinatesz-_grid_sampler_2d.<locals>.compute_coordinates  sx    1MQ;;vq$(33#6vq!tax.#Q #6vr1t8a<#P ;;/D1H==r,   c                 (     | |      } ||      S r5   r+   )r
  r   	coords_unr  r  s      r-   compute_source_indexz._grid_sampler_2d.<locals>.compute_source_index  s    -	"9d33r,   r$   r#   r  ysc                     t        j                  d| k  t        j                  | k  t        j                  d|k  |k                    S rp   )re   r  )r  r   iHiWs     r-   in_bounds_condz(_grid_sampler_2d.<locals>.in_bounds_cond  sF      GU&&rBw0A0A!r'2PR70ST
 	
r,   r  wsc                      | |      rndt        	
fd| j                  t        j                        |j                  t        j                        |fD              S )Nr#   c              3   p   K   | ]-  }t        j                  |d       j                         / ywr  )re   rf   rT  )r  rx  r  r  r  oHoWs     r-   r  z1_grid_sampler_2d.<locals>.clip.<locals>.<genexpr>  s7      
 KKa#((Ar26
s   36r   )r  r8   re   r  )r  r   r%  r  r  r  r  r  r$  r(  r)  s      @@r-   clipz_grid_sampler_2d.<locals>.clip  sY    b"%
 A1 
ee%++e.EKK0H"M
 
 	
r,   ixiyc                 8     	| ||      \  }}}||f   |z  S r5   r+   )
r+  r,  r  idx_xidx_yw_C_idxN_idxr  r*  s
         r-   get_summandz%_grid_sampler_2d.<locals>.get_summand  s0    B?ubue+,r11r,   ).r   ).r#   r   c              3   :   K   | ]  \  }}} |||        y wr5   r+   )r  r+  r,  r  r3  s       r-   r  z#_grid_sampler_2d.<locals>.<genexpr>  s(      
R B"
s   c                 <     |       } |      } ||d      S rX   r+   )r+  r,  r9   rV   r  r3  r"  r#  s       r-   get_value_boundedz+_grid_sampler_2d.<locals>.get_value_bounded;  s*    #B+A#B+Aq!Q''r,   r  c                     | dz
  z   } dz
  |       |       dz   |       dz   |      f}t        |      S )Nr#   r$   )r  )r  iy_ofscsr6  ix_nwiy_nwtxs      r-   	get_coeffz#_grid_sampler_2d.<locals>.get_coeff@  s[    cAg&F!%!)V4!%0!%!)V4!%!)V4	B ,B33r,   c              3   .   K   | ]  } |        y wr5   r+   )r  r  r=  s     r-   r  z#_grid_sampler_2d.<locals>.<genexpr>J  s     :#y~:   r5  )re   r   r   r  r  rT  r  r  r  r   r  r  r  rQ   r  rP   r  ).r  r  r  r  r  r  r  rR   twor9   rV   r+  r,  ix_neiy_neix_swiy_swix_seiy_sew_nww_new_sww_se
ix_nearest
iy_nearesttyr  r  r1  r  r2  r*  r  r=  r3  r6  r"  r#  r$  r:  r;  r(  r)  r  r<  r  s.   ` ````                     @@@@@@@@@@@@@@@@@@@r-   _grid_sampler_2drN    s    
LLi'B 
LL	!#Q"F "# "& "

F 

s 

 

PV 


>F 
># 
>& 
>4V 43 46 4 77LAq"bZZNAr2s!8O8 yyAr2s+221aRC
6 
v 
& 

 LL188,11!Q1=ELL188,11!Q1=E

 

V 

 

4F 

 

2 2F 2& 2 2
 	VAVAQ!!R(!!R(xxz288:uqy%ueaiueu
urz*U
urz*
rEz*U
rEz* 
 t$t$t$t$	 
 
 	
 
q	 !!R(!!R(XXZ
XXZ
:z155BB

%Z%ZaBaB	(& 	(f 	( 	( 	(
	43 	46 	4 	4 :q::'33r,   c                 "    t        | ||||      S )N)r  r  r  r  )rN  )r  r  r  r  r  s        r-   grid_sampler_2drP  N  s      	-!# r,   c                 &    t        j                   j                         dk(  xr j                         dk(   fd       t        j                   j                  d      j                  d      k(   fd        z  j	                  d      S )Nr$   r#   c                  L    d j                          dj                          S )Nzmatrix @ vector expected, got r  r}   rx   r  s   r-   r~   zmv.<locals>.<lambda>g  s!    0BswwykJ r,   r   c                  v    d j                  d       d j                  d       dj                  d       dS )Nzsize mismatch, got input (r   r9   r#   z), vec (r0  r  rS  s   r-   r~   zmv.<locals>.<lambda>k  s<    ,TYYq\N!DIIaL>RURZRZ[\R]Q^^_` r,   r}   )re   r   rK   r   r   rS  s   ``r-   r  r  a  ss     
LL
a*CGGINJ 
LL		!#` 3J""r,   c                     |-|dz
  |z  dz   }d|z
  | z  |t        j                  |       z  z
  }nd|z
  | z  t        j                  |       z
  }|||z  }t        ||      S rX   )r   
logsigmoidr   )rx   r   r   
pos_weightr   
log_weightr   s          r-    binary_cross_entropy_with_logitsrY  p  sv    
  1n.2
F
d"j1<<3E&EFF
d"Q\\$%77f}i00r,   tensor1tensor2is_outc           
         | j                   |j                   k\  r| |fn|| f\  }}ddlm} |j                   dk\  r|j                   dk  sy|j                  r|sy| j                   dk(  ry ||j	                         dk(        ry|j
                  }|j                         }t        d t        |d d |d	d
 |d	d
       D              S )Nr   )guard_size_obliviousr   r$   FTc              3   4   K   | ]  \  }}}|||z  k(    y wr5   r+   )r  st1st2s2s       r-   r  zshould_fold.<locals>.<genexpr>  s&      S#r 	sRxs   rM  r#   rO   )	rn  rw  r^  rw  r   r  rg  r  r  )rZ  r[  r\  t1t2r^  t1_shape	t1_strides           r-   should_foldrg    s     $+<<7<<#?gwgwEWFBJGGqLRWW\	||qBHHJ!O,xxH		I !)CR.)Ab/8Ab>R  r,   )pass_is_out)r\  c                   | j                         }|j                         }|dk7  r|dk7  sJ |dk(  r|dk(  rt        j                  | |      S |dk(  r|dk(  rt        j                  | |      S |dk(  rC|dk(  r>t        j                  t        j
                  t        j                  | d      |      d      S |dk(  r|dk(  rt        j
                  | |      S t        | ||      r||kD  }|r|j                  n| }|s|n|dk(  r| j                         n| }|j                  }t        |d d       }	t        t        j                  |	      }
|j                         dk(  }|r|	j                  |j                  d          |j!                  |
|d         }|r>|j                  |      j#                  |	      }|r|j                  j%                         S |S |j                  |      j#                  |	      S |dk\  r^|dk\  rX|dkD  r| j'                  d      nd}| j'                  d      }| j                  d d }|dkD  r|j'                  d      n|j'                  d      }|dkD  r|j'                  d      nd}g }t)        |dz
        D ]"  }|j                  |j'                  |             $ |dk(  rn|dk(  ri|d   |d   k7  r^|d   dk(  r'| j*                  rt-        | j	                  d      |      S |d   dk(  r'|j*                  rt-        | |j	                  d            S t        t        j.                  ||            }|||gz   }t1        |      }| j3                  |      j!                  |||      }|dk(  }|r7||gz   }|j3                  |      j!                  ||      j                  d      }n)|||gz   }|j3                  |      j!                  |||      }|}	|dkD  r|	j                  |       |dkD  r|	j                  |       |r/|j5                  |      j	                  d      j#                  |	      S |j5                  |      j#                  |	      S t        j6                  dd        y )	Nr   r#   r$   rO   rM  r   Fc                       y)Nz/both arguments to matmul need to be at least 1Dr+   r+   r,   r-   r~   zmatmul.<locals>.<lambda>  r  r,   )rK   re   dotr  r  r  rQ   rg  rR  rx  r  rf  r   operatorr   rS  r  rT  r  r   rP   rw  rQ  broadcast_shapesr  r  bmmr   )rZ  r[  r\  dim_tensor1dim_tensor2r@  rc  rd  sizes_1output_shapefolded_dim1t2_is_matrix	t1_foldedr   r  m1batch_tensor1m2rE  batch_tensor2rM  expand_batch_portiontensor1_expand_sizeexpand_batch_producttensor1_expanded
vector_rhstensor2_expand_sizetensor2_expandeds                               r-   rQ  rQ    sG    ++-K++-K!q 000aK1,yy'**		kQ.xx))		kQ.}}UXXeoogq&A7KQOO		kQ.xx))	Wgv	.  +-	$WZZ'$G+:J799;PW 	 ((GCRL)X\\<8 vvx1}, JJ{GBK8	 \\"%**<8F-6699'')BFB<<#((66		kQ. !,aGLLQ\\"cr*!,qW\\"gll26F +aGLLQ#%{Q' 	2A  a1	2 1q a M!$44Q1$)>)>gooa0'::Q1$)>)>gwq'9::  $""=-@ 
 3aW<#$89 #>>*=>FF !R
 !A%
"6""=23-r21  #7"a"@&~~.ABJJ$b!  ,?"?"#''(89AA"EJJ<XX#''(89>>|LLUUVr,   r  r  c                      !  j                   \  }}|\  }}dd}	d }
 |	|||      } |	|||      }t        j                  | j                        j	                  |ddd      t        j                  | j                        j	                  d|dd      t        j                  | j                        j	                  dd|df      }t        j                  | j                        j	                  ddd|f      } |
|||      }|j                         }||z
  !|j                  t        j                        } |
|||      }|j                         }||z
  }|j                  t        j                        }|dz
  ||dz   |dz   f}|dz
  ||dz   |dz   f fd  !fdt        fd	|D              }t        ||      }t        j                         }|j                  |
      }|S )Nc                 L    |r|dkD  r| dz
  |dz
  z  S dS |
|dkD  rd|z  S | |z  S r  r+   r  s       r-   compute_scalez1upsample_bicubic2d_default.<locals>.compute_scale  sB    5=\GaKHqL1HqH % 1eai1u9WWxEWWr,   c                 &    |r| |z  S | |dz   z  dz
  S r  r+   r  s      r-   r  z8upsample_bicubic2d_default.<locals>.compute_source_index$  s$    9$$IO,s22r,   r  r#   r   r$   c                     t        j                  | ddz
        }t        j                  |ddz
        }t        j                  ||g      S r  )re   r   r   r  )	r   r  y_idxx_idxr1  r2  r  r"  r#  s	       r-   load_boundedz0upsample_bicubic2d_default.<locals>.load_bounded?  sK    B26*B26*!!!eUE5%ABBr,   c                 H     t         fdD              }t        |      S )Nc              3   0   K   | ]  } |        y wr5   r+   )r  x_ofsr  rV   s     r-   r  zCupsample_bicubic2d_default.<locals>.get_x_interp.<locals>.<genexpr>E  s     EEa/Es   )r  r  )rV   coeffs_xixs_ofsr  t_xs   ` r-   get_x_interpz0upsample_bicubic2d_default.<locals>.get_x_interpD  s     EWEE'#66r,   c              3   .   K   | ]  } |        y wr5   r+   )r  y_ofsr  s     r-   r  z-upsample_bicubic2d_default.<locals>.<genexpr>H  s     >U\%(>r?  rK  r5   )r  re   r  r  rT  r  r8   r  r  r  rA   r  r  )"r  r  r  r  r  r  r  r(  r)  r  r  height_scalewidth_scaleout_yout_xreal_xin_xr+  real_yin_yt_yr,  iys_ofscoeffs_yr  rL  r1  r2  r  r"  r#  r  r  r  s"   `                         @@@@@@@@r-   upsample_bicubic2d_defaultr    s    77LAq"bFBX3 !R@LBw?KLL188,11!Q1=ELL188,11!Q1=ELLAHH-22Aq"a=AELLAHH-22Aq!R=AE!+umDF<<>D
4-C	u{{	#B!,}EF<<>D
4-C	u{{	#BAvr2626*GAvr2626*GC C
7 >g>>H%h4F //2M];FMr,   r  c                 $   t        j                  t        |      t        |      z   dk(  d        |H|J t        t        t
        t
        f   t        d t        | j                  dd  |      D                    }|r|nd\  }}t        | ||||      S )Nr#   c                       y)Nz:Must specify exactly one of output_size and scale_factors.r+   r+   r,   r-   r~   z(upsample_bicubic2d_vec.<locals>.<lambda>^  r  r,   c              3   P   K   | ]  \  }}t        t        |      |z           y wr5   )r   r   )r  r  rk   s      r-   r  z)upsample_bicubic2d_vec.<locals>.<genexpr>d  s*      Au 	!u,-s   $&r$   r  )
re   r   r'  r
   r   r  r  r  r  r  )r  r  r  r  r  r  s         r-   upsample_bicubic2d_vecr  Q  s     
LL[D//14L (((#s(O  #AGGABK ? 
 )6}<GW%amWgVVr,   c                 (      fd}t         ||      S )Nc                     t        j                  |  ||z   j                        }|dz
  |dz
  |j                         z
  j                         z
  S )Nr  r#   )re   r  r  r   r   middler   dim_idxr  s       r-   r  z_reflection_pad.<locals>.idxs  sF    ,,ufunQXXFzVaZ'++-7<<>>>r,   _reflection_or_replication_padr  r  r  s   `  r-   _reflection_padr  m  s     ? *	 r,   c                 (      fd}t         ||      S )Nc                     t        j                  |  ||z   j                        }t        j                  |d|dz
        S )Nr  r   r#   )re   r  r  r   r  s       r-   r  z_replication_pad.<locals>.idx  s6    ,,ufunQXXF{{7Avz22r,   r  r  s   `  r-   _replication_padr  ~  s     3 *	 r,   idx_fnc                 n   t        |      dz  t        j                  | j                         dz   dz   fv fd       | j                   d  }| j                         z
  }t              D cg c]  }|ddz
  |z
  z      }}t              D cg c]  }|ddz
  |z
  z  dz       }}| }t              D ]E  }d g|j                         z  }	 |||   ||   ||         |	||z   <   t        j                  ||	      }G t        j                  |      }
|j                  |
      }|S c c}w c c}w )Nr$   r#   c                  (    d  d dz    d dz    dS )Nreflection_padzd requires r#   zD or r$   zD inputr+   r}   s   r-   r~   z0_reflection_or_replication_pad.<locals>.<lambda>  s$    .[q	sQwiwO r,   rK  )rs  re   r   rK   r  rP   r   r  rA   r  r  )r  r  r  	inp_shapenc_dimrM  padding_leftpadding_rightr  r  rL  rK   s              @r-   r  r    sH   
 g,!
C	LL	C!GS1W%%O IUUWs]F8=c
C1GAq1-.CLC=B3ZHWQ#'A+.23HMHF3Z 1&**,. a)A,a@PQAJ##FC01 //7M];FM DHs   1D-D2r   r   r  c                j    t        j                  | ||      }t        j                  | ||      }||fS )Nr  )re   aminr4  )rx   rK   r  r  r4  s        r-   aminmaxr    s2     ::dW5D::dW5D:r,   r   c                    t         j                  t        j                  t        j                  |       d|       |||      S )Nr   r   )r   r   re   rf   isnan)rx   rK   r  r   s       r-   nansumr    s2     88EKKD 11d;S'QV8WWr,   r   r  r  r  r  c          	      N    t         j                  j                  d| d||||      S )Nr   r#   r  r   r  
start_step)r^  r   r  r  r  s        r-   arange_defaultr    s/     ;;!!	3vf "  r,   c          	      N    t         j                  j                  | |d||||      S )Nr#   r  r  )r]  r^  r   r  r  r  s         r-   arange_startr    s/     ;;!!sAU6&Z "  r,   c                      ddl m}  || i |S )Nr   )out_dtype_dense)!torch._higher_order_ops.out_dtyper  )rC   rD   r  s      r-   out_dtype_decompr    s    AD+F++r,   marginc                 t   	
 t        j                          t        j                         j                  d   
 j                  d   	t        j                  |dk(  xs |dk(  d        t        j                   j
                  dk(  xr 	dk7   fd       t        j                  j
                  dk(  xr j                         
k(  
fd       Qt        j                        t        j                  j
                  dk(  xr j                         	k(  	fd       j                  d      t        j                   d      }||z
   z   }|j                  d      }|dk(  r|n||z  }|   z  }t        j                  	 j                  	      }t        j                  |k7  |d      }|t        j                  j                  k(  r|j!                         S |t        j"                  j                  k(  r |j%                         |j                  d   z  S |j!                  d
      S )Nr   r#   r$   c                       y)Nz only p == 1 and p == 2 supportedr+   r+   r,   r-   r~   z#multi_margin_loss.<locals>.<lambda>  r  r,   c                  "    d j                    S NzMExpected non-empty vector or matrix with optional 0-dim batch size, but got: r  )r   s   r-   r~   z#multi_margin_loss.<locals>.<lambda>  s    _`e`k`k_lm r,   c                  (    d  dj                    S )Nz#inconsistent target size, expected r  r  )nframer   s   r-   r~   z#multi_margin_loss.<locals>.<lambda>  s    5fXYv||nU r,   c                  (    d  dj                    S )Nz#inconsistent weight size, expected r  r  )rK   r   s   r-   r~   z#multi_margin_loss.<locals>.<lambda>  s    9#i~V r,   r  r  r}   )re   
atleast_2d
atleast_1dr  r   rn  r   rQ   r  rS  r  r  rf   r"   r)   ry   r   r*   r   )r   r   rE  r  r   r   urg   r  rK   r  s   ``  `    @@r-   multi_margin_lossr    s    U#Ef%F[[^F
++a.C	LLa!16#MN	LL

a$C1Hm 
LLq5V\\^v5U !!&)KK163!6V	
 a FU0A
UA	AA!VQAv
,,s5<<
0CC6M1a(AINN(((vvx	imm))	)uuw##vv!v}r,   	is_targetc                    | j                   |j                   t        j                  |       } t        j                  |      }| j                   d   }t        j                  t	              dk  xr |dk7  fd       t        j                  t	              dk  xr k(  fd       t        j
                  ||j                        }|dk(  }t        j                  t        j                  |||      dd	      }||k  }t        j                  ||d      }t        j                  | d|
      }	t        j                  ||d      }
t        j                  ||
j                  d      k(  d      }d|	j                  j                  d      z
  | z   }|j                  d      }||z  }t        j                  |d|      }|t        j                  j                   k(  r!|j#                  d      j%                         }n@|t        j&                  j                   k(  r|j#                         }n|j#                  d      }|j)                  | j*                        j-                        }||fS )Nr#   r$   r   c                      d  S r  r+   )orig_input_shapes   r-   r~   z0multilabel_margin_loss_forward.<locals>.<lambda>  s    _`p_qr r,   c                      d d  S )Nzinconsistent target size: z for input of size: r+   )r  orig_target_shapes   r-   r~   z0multilabel_margin_loss_forward.<locals>.<lambda>   s    ,->,??STdSef r,   r  rO   Tr  r  r}   rc   )r   rO   )r  re   r  r   rs  r  r  r  rf   r  anyrQ   TrS  r"   r)   ry   r   r   r*   r8   r   r  )r   r   r   rK   r  is_endend_idxtarget_masktidx0r  tidx1r  rg   r  r  s                @@r-   multilabel_margin_loss_forwardr    s    {{U#Ef%F
++a.C	LL"/saxr 
LL!#M(9=M(Mf
 ,,s6==
1Cr\FjjVS#6BMG-KKKVQ/EU%0AKKVR0E		#R!88a@Iaccmmm##e+A	AA	CAIq!$AINN(((EEgE##%	imm))	)EEGEEgEU[[)112CDIi<r,   )	attn_maskrk   querykey	dropout_p	is_causalr  c          
           j                   }t        j                  t        j                          fd       t        j                   j	                         dk(  xr( j	                         dk(  xr j	                         dk(   fd       t        j                  dk(  fd       t        j                   j
                  d   j
                  d   k(  xr j
                  d   j
                  d   k(  d        t        j                  j                   ||d |      \  }}	|j                  d	d
      j                  t        j                        }|j                  d	d
      |	fS )Nc                  "    d j                    S )Nz-query must be FP32, FP64, BF16, FP16 but got r   )r  s   r-   r~   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>Z  s    ?}M r,   r5  c                  n    dj                          d j                          dj                          S )Nz,q, k, v must be a 4 dimensional tensor, got r  r}   )r  r  ry   s   r-   r~   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>^  s3    >uyy{m2cggi[XZ[`[d[d[fZgh r,   r   c                      d  S )Nz&dropout probability must be zero, got r+   )r  s   r-   r~   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>a  s    $J9+"V r,   r   c                       y)Nz&q, k, v should have the same head sizer+   r+   r,   r-   r~   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>e  r  r,   )r  r  r  dropout_maskrk   r#   r$   rK  )r   re   r   r(  rK   r  r   "_scaled_dot_product_attention_mathr  r@  r  rP  )
r  r  ry   r  r  r  rk   r   r   attns
   ````      r-   *scaled_dot_product_flash_attention_for_cpur  L  s@    KKE	LL&M 
LL		q@SWWY!^@		q0@h 
LLSV 
LLA%++a.(KSYYq\U[[^-K8
 ::BB C 	LFDV a#..U=T=T.UFQ"D))r,   c                 .    t        |       fd       }|S )Nc                  <     | i |}| d   j                  |      S rp   )r   )rC   rD   r   outplace_ops      r-   
inplace_opz$register_inplace.<locals>.inplace_op  s%    4*6*Aw}}S!!r,   r   )aten_opr  r  s    ` r-   register_inplacer    s"    G$" %" r,   c                 B   | j                         s&| j                         st        |      }t        |      }t        j                  ||      }t        |t        j                        r|dk7  r||z  }|dk(  r|S t        |t        j                        r|dk7  r| |z  } | |z   S r  )r(  r  r  re   rn  r7   numbersNumber)rx   batch1batch2r_   rj   r  s         r-   baddbmmr    s     !!#DOO,=4yE
YYvv&FeW^^,
%qydGNN+tqyd{&=r,   c                 2    t        j                  | |d      S )Nr  rO  rQ  )rx   rD  s     r-   floor_divider    s     99T588r,   c                 `    t        j                  t        j                  | j                  d      S rX   )rH   r   rl  r   r  )rx  s    r-   	sym_numelr    s    HLL!''155r,   r   r   c                    |"t         j                  j                  | g |      S t         j                  j                  | g ||      S )Nr   r  )r   r   dim_IntListIntList_out)rx   r   r   s      r-   sum_defaultr    sC     {xx##D"E#::xx##D"Es#CCr,   c           	          |@t         j                  j                  | t        t	        | j                                           S t         j                  j                  | |g      S r5   )r   r  dimsrf  rP   rK   )rx   rK   s     r-   squeeze_defaultr    sG    
{||  tE$((*,='>??||  u--r,   c                     t        fdt        t        | j                              D              }| j	                  d|d      }| ||z  z  |fS )Nc              3   .   K   | ]  }|k7  s	|  y wr5   r+   )r  rM  rK   s     r-   r  z)_weight_norm_interface.<locals>.<genexpr>  s     @1qCxQ@s   
r$   Tr0  )r  rP   rs  r  r   )r9   rV   rK   keep_dimr   s     `  r-   _weight_norm_interfacer
    sI     @c!''l 3@@H66!Xt6,DD>4r,   assume_uniqueinvertc                   t        | t        j                        s!t        j                  | |j                        } t        |t        j                        s!t        j                  || j                        }|j                         dt        | j                         d      z  k  rt        | ||      S t        | |||      S )Nr  g      $@g(\?r  r  )	r7   re   r   rV  r  r   rN  isin_defaultisin_sorting)elementstest_elementsr  r  s       r-   isinr    s     h-<<1E1EFmU\\2]8??Ktc(..*:E&BBBHmFCCm=
 	
r,   r  c                N   | j                         dk(  r%t        j                  | t        j                        S  | j                  g | j
                  d|j                  z   }|s||k(  }n||k7  }t        t        d|j                   dz
  d            }|j                  |      S )Nr   r   )r#   rO   r#   r}   )
r   re   
empty_liker'  rT  r  rn  r  rP   r  )r  r  r  r9   cmprK   s         r-   r  r    s    ~~1

;;Dx~~D0B0B)BDA= = 
b=---126
7C77s7r,   c                   | j                         }|j                         }|rt        j                  ||g      }t        j                  |d      \  }}|dd  |d d k(  }	t        j                  |	ddgd      }	|r|	j                         }	t        j                  |	      }
|
j                  d||	      }
|
d| j                          S t        j                  |      \  }}t        j                  ||      }t        j                  ||j                         k  |d      }||   |k(  }|r|j                         n|}|j                  | j                        S )NT)stabler#   rO   r   F)r  re   r"  sortrQ  logical_notr  r  r   searchsortedrf   r  r  )r  r  r  r  elements_flattest_elements_flatall_elementssorted_elementssorted_orderduplicate_maskr  sorted_test_elementsrR   r  test_idxr  s                   r-   r  r    s?   $$&M&..0 yy-1C!DE(-

<(M%(,0DD..~1vuM+779N/q,?A())"'**-?"@a  !5}E;;s%9%?%?%AA3J"8,=#)coos{{8>>**r,   c                 .    | j                  d      }||   S rN   )r  )rx   ru  	flatteneds      r-   taker'    s     R IUr,   )F)none)g      ?gUUUUUU?FN)FN)r$   )r   NNr#   r5   )rO   FFr  r  r  )r#   r#   F)Fr   )r   rc   N)r   r#   Nr  )NNN)r   r   FT)r   r   Fr  )r   F(s  rH   r  rl  rh  enumr   r   r   	itertoolsr   r   typingr   r	   r
   r   r   r   r   r   re   torch._primsr  r  torch._prims_common_prims_commonrA   torch.nn.functionalrh  r  r   r   r   r   torch._decompr   r  r   r   r   r   r   torch._prims_common.wrappersr   r   r   r   torch.utilsr   r?   torch.utils._pytreer   rr  DispatchKeyr    str__annotations___opsr  r   r"   r2  r'  rJ   r3  compute_only_pw_cast_for_opmathpw_cast_for_opmathr  pw_cast_for_int_to_realr  rS   r\   r^   rh   r  rt   fillScalarrz   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  py_implAutogradCUDA	Generatorr   r   r   r   r   r   r   r)   ry   r   r  r  rT   r  r  r   r  r  r*  r3  r6  r<  r?  rA  rC  rG  r[  rc  slicert  rx  r~  r  r  r  r  r  r  r  r  r  r#  CompositeImplicitAutogradAutogradr"  r7  r;  rB  rK  r  rZ  r]  ra  rl  rn  r|  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  unsafe_chunkr	  r  r  no_statsr  r  _fused_dropoutr  r  r#  r  rL  detachlift
lift_freshr&  r+  r.  r@  rB  rF  _adaptive_avg_pool2dru  r|  r~  rz  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r	  r  r  r  r'  r*  r.  r>  rG  rnn_tanhr   rN  rnn_relurQ  rV  rW  rZ  rf  ri  ro  r|  lstmr  r  r  r  grur  r  r  r  r  r  r  r  r  r  r  r  r  r  r  _unsafe_viewr  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  rN  rP  r  rY  rg  rQ  upsample_bicubic2dr  r  reflection_pad1dreflection_pad2dreflection_pad3dr  replication_pad1dreplication_pad2dreplication_pad3dr  r  r  r  r  r  r  r  r]  r  r  r  r  +_scaled_dot_product_flash_attention_for_cpur  r  r  r  r  r   r  r  rK   r  r
  r  r  r  r'  addbmm_addbmmaddmm_addmv_baddbmm_fill_gelu_r  
hardswish_	hardtanh_hardtanhhardsigmoid___iand____and____ilshift__
__lshift__r  r  index_reduce_index_reduce__ior____or____irshift__
__rshift____ixor____xor__leaky_relu_r   logit_logitrelu_r  renorm_renormround_r  scatter_r  scatter_add_scatter_addscatter_reduce_scatter_reducesilu_r+   r,   r-   <module>r{     sE      
  % $ N N N   #   , , 0 7 S S  * (hh"" c zz~~   %!!99! !H #*88@@# 
 uDDLL  "uDDQQ  c f  **+\2F 2v 2   ,2 --.\4v 4& 4   /4 ../\S S6 S S5 S   0S
 ))*\


 
 	

 
 
   +
2 ))*+( ,( ))*+"V " ," (()@f @ @   *@ 112\f F    3 ../\PP%P05P@EP  0P 'GF Gv G   (G //0F & V    1 //0\:F :& :U :  1: 001\LL%L7<LNRL   2L **+\) )f )3 )   ,)< **+5v 5f 5  ,5 		"&v && &   #& **+\>v >V > >   ,>
 **+6 6 66 6 ,6 334%%
% % 66>	% 5% --.&&{'?'?@ %+/5
55 5 	5
 5 (5 5   A /5( ../''(@(@A +/X
XX X 	X
 X (X X  B 0X 667



 
 	

 
 
 
 
   8
$ 112\<f <F <F <v <   3<v #   &3<>>3G3G1
1 1-011   '1 ../\11 &1061CF1   01 ++, ^^))	1
11 1 	1   -1 44<<=%/5BEMR  > 44??@
R
R

R 
R 	
R
 
R 
R  A
R 00889		%	/5	BE	NS	  :	 00445
R
R

R 
R 	
R
 
R 
R  6
R$$
$ $ V	$
 $ $ $ $@ ))*\Nf NF N N N   +N$ ../\''
' ' V	'
 ' ' ' '  0'T 001\
  V	
      2B 112  $^^))	1
11 V1 	1
 1   31& ::;\
  $^^))
  V	
     <  --. ^^))111 1 	1   /1 667\
 ^^))			
	 	 		
 	   8	 		") )v )% )  #) ,,-& &F &v &  .& ++,	O	Oc	O 
	O 		O
 
	O 	O  -	O 

))* /?
/? 
/? C=	/?
 
#/? /? +/?d ,,-E Ed3i Ec ERU E  .E
 ../OO&*3iO9<ODGOORO  0O%+:?++ 334\ XX!'X.1X@EX !  5X 778 KK!'K.1K@EK !  9K*& $MMcM 3iM #Y	M
 IM M  %M` $]]c] c] 3i	]
 #Y] I] ]   %]@ 445
 
v 
e 
  6
 ,,-Y
Y"3iY47Y?BYJMYY  .Y" ++334>B

%
,4UO

  5
& %kCCDk2236 e HTN  4 E & ++,VVA& AU A8D> A  -A & S    '* ))*F  T   +, ' $  	
    (& 556  	
   7:DI &\	  
&\	*T&\ 
tF| 
# 
"&\"	" "J 00$//2E2EFG
 !	&\	  
&		
  H --.56
#Cy/2	&\ /B 	'')C)C)G)GH "&	
c 
 
$v,		
 d6l  ))0015 5C 5c 5%PSBT 5 25 44<<=67BB $S	B03B
63;B >B 

))*/ /C /c /%:L / +/" --55)) /
/ &/ 
/ 63;	//. 

# f F # #    $" ../
 

  	
     0" 

# f 6      $ 77??@S&S&S& S& 	S&
 FS& S& S& 
S& S& dS& 8FXf-x/??@S&  AS&n 77;;<  	
 F   
  d ,, ,, ,, 8FXf-x/??@ =88F# x/?  77??@JJJ 3iJ 	J
 J VJ 6
J dJ 8FXf-x/??@J AJ\ 77;;< 3i 	
  V 6
 d ,, ,, ,, 8FXf-x/??@ =4OOVO 6
O 6"	O
 &!O O O 
O O 6668F#3Xf5EEFOd ../UK/((V( 6
( 6"	(
 &!( ( ( 
( 666!"( 0 0(4 ''(<(<=''(M(MN 
 
V 
 6
 
 6"	 

 &! 
  
  
 
 
 666!" 
 O > 
F "";#H#HIG4< G JG AAIIJV 6
 	
   
 666!" K* 55==>((V( 6
( 	(
 ( ( ( 
( 666!"( ?(  55>>?((V( 6
( 	(
 ( 
( 666!"( @( @@HHIKKVK 6
K 	K
 K K K 
K 6666612K JK0 ++,VV   - & $(%)37 EKK 
 U\\"   E//0  'H diiAB  C &&{';';<--.VVVV, 6
 6"	
 &!  !&  - / => 77??@``` V` 6"	`
 &!` ` &!` ` 
` d` 68F#Xf%556` A`H 77;;<""" V" 6"	"
 &!" " &!" " 
" d" ,," ,," ,," 68F#Xf%556" ="J 667VVV$  6"	
 &!  v   % 82 112c'v c'E#s(O c'   3c'L ( HH	H H 	H H )H ' II	I I 	I I  (I$ %@%@	%@ %@ 	%@ %@ %@P ))112"";#H#HI J 32 (()<: <C <
 <J < *< (=* =3 =z =: =  )=@@@$.@8B@PT@, 001Xx (f (vv~)> (  ! 2( % $'$'+/		tS%	  c5 
! (	  &  &; ';
8 //334$$[%J%JK$$[%9%9:@ ; L 5@ 66::;##++K,Q,QR##++K,@,@AG B S <G //334$$[%J%JK$$[%9%9:K ; L 5K 66::;##++K,Q,QR##++K,@,@AR B S <R //334$$[%J%JK$$[%9%9:T ; L 5T 66::;##++K,Q,QR##++K,@,@A B S <@ //778  (()=)=> #>>c> UO> 	>  ? 9> 66>>?''//0D0DE #>>c> UO> 	>  F @> //778  (()=)=> !% $		C	Cc	C uo	C uo		C
 	C  ? 9	C 66>>?''//0D0DE !% $		C	Cc	C uo	C uo		C
 	C  F @	C //778  (()=)=> !% $ $c uo uo	
 uo   ? 9 66>>?''//0D0DE !% $ $c uo uo	
 uo   F @ >/ FK+\&,/-d& R ++,[BBC[112. 3 D -.8 ++,[BBC[112. 3 D -.8 **+KAABK001. 2 C ,.@ **+KAABK001. 2 C ,.@/6<~/d 		(>>?--.S / @ )S> 		'==>,,-S . ? (S<;; &{<<={++,. - > '.6 '==>,,-. . ? (.6 44889!!))+*O*OP!!))+*>*>? @ Q : 33778  (()N)NO  (()=)=> ? P 9 0044511556##K$I$IJ##K$8$89%%k&K&KL%%k&:&:;&&{'L'LM&&{';';<A = N < M : K 7 6A //779O9O9S9STU
 !%	KKcK K uo	K
 K  VK 	%%t'?'?'C'CD !!))+*>*>?
 !% $UUcU U uo	U
 uoU U  @
U 	&&(A(A(E(EF 
 !% $ $

c
 
 uo	

 uo
 uo
 
 
V/ EEcE E %!	E
 E ER ))112F v $  3 ,,d.?.?@A  B ++,-" ."2 
2 2  V2  	2 
 2  66>2 j --.X~&L
LL VL 	L
 L 66>L ' /L4 //0X~&L
LL VL 	L
 L 66>L ' 1L/F /u / /5F 5u 5 5 3E F%7 FV F F!Xf% !& !NN#'N05NEJ\\N&f & & &T &"/f / / / /T /"!V !49 !T !$V $49 $T $ 223S StCy S S   4S  c4c4
c4 c4 	c4
 c4 c4 c4L ,,-  
  	
     .   	#   !	# ==>$)..:N:N1  ?1  d t 0 [BBC', tW  DtWn //778
  $#::sCx: : e_	:
 e_: :  9:z //334$$[%J%JK$$[%9%9:
 48	WW%S/*W W E%,/0	W
 W   ; L 5
W. --.--.--.	v 	c3h 	F 	   / / /
	 ../../../	 	sCx 	V 	   0 0 0
	38_ c3_f,- 	8 %UEu   & $X4 X  %X ,,dkkoo>? $( ==%)
	
 EKK 
 LL	

 U\\"
 
  @
 **+,
 $( ==%)	 EKK 	
 LL U\\"  - 	", #, ../''(<(<= #^^))))) ) 	)
 V) ) )  > 0)X ;;<$$,,44[5I5IJX{#--- - 66>	- $ K =-x HHPPQ
 G* #'!G*G*	G* G* 	G*
 G* G* E?G* 66>G* RG*T '   ( ))*9  +9 '6 (6 ))488<<89 $( 		D
	D EKK 	D 
&			D
 	D :	D --t||/?/?@A.& .x} . B. 		==>  ?  		"38 
  #
 5: 
 <A +: 		"  #
 t{{ + djj ) djj )  - TYY ' TYY ' $.. 1  / ""D$4$4 5  - !!4?? 3 $.. 1 ##T%6%6 7 t{{ + !!4?? 3  - !!4?? 3 djj ) TYY ' t{{ + djj )  - ""D$4$4 5 %%t':': ; TYY 'r,   