
    '}hO                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZ d dlZd dlmZ d dlmZmZ d d	l m!Z! d d
l"m#Z# ddl$m%Z%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1m2Z2m3Z3 ddl4m5Z5 ddgZ6e G d d             Z7e G d d             Z8dZ9 G d de      Z: G d de:      Z; G d de:      Z<de.de=fdZ>d e=d!ee.   deee.      fd"Z?d#ej                  d$eej                  ej8                  f   d%e.d&eBde3f
d'ZCd(ed)ej                  d*ej                  d+e-d,e=d-eEd.e=ddfd/ZF G d0 d1e      ZG G d2 d3eG      ZH G d4 de2      ZI G d5 de1      ZJy)6    N)ABCabstractmethod)contextmanager)	dataclass)Path)CallablecastDict	GeneratorIOIterableIteratorListOptionalTupleUnion)Tensor)_get_available_device_type_get_device_module)narrow_tensor_by_index)Future   )MetadataMetadataIndex)LoadItemTypeLoadPlanLoadPlannerReadItemSavePlanSavePlanner	WriteItemWriteItemType)StorageReaderStorageWriterWriteResult)_create_file_viewFileSystemWriterFileSystemReaderc                   0    e Zd ZU dZeed<   eed<   eed<   y)_StorageInfoz#This is the per entry storage info.relative_pathoffsetlengthN)__name__
__module____qualname____doc__str__annotations__int     f/var/www/html/test/engine/venv/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.pyr*   r*   1   s    -KKr6   r*   c                       e Zd ZU eed<   y)_StoragePrefixprefixN)r.   r/   r0   r2   r3   r5   r6   r7   r9   r9   :   s    Kr6   r9   z.distcpc                   v    e Zd Zedededdfd       Zedd       Zedee	e
j                  ef      fd       Zy)	_TensorLoadersizeobjreturnNc                      y Nr5   selfr=   r>   s      r7   addz_TensorLoader.addC       r6   c                      y rA   r5   rC   s    r7   start_loadingz_TensorLoader.start_loadingG   rE   r6   c                      y rA   r5   rG   s    r7   valuesz_TensorLoader.valuesK   rE   r6   r?   N)r.   r/   r0   r   r4   objectrD   rH   r   r   torchr   rJ   r5   r6   r7   r<   r<   B   sk     & T     u||V';!<=  r6   r<   c                   h    e Zd ZdeddfdZdededdfdZd
dZde	e
ej                  ef      fd	Zy)_SerialCpuLoaderresolve_funr?   Nc                      || _         g | _        y rA   )rP   items)rC   rP   s     r7   __init__z_SerialCpuLoader.__init__Q   s    &/1
r6   r=   r>   c                 >    | j                   j                  ||f       y rA   )rR   appendrB   s      r7   rD   z_SerialCpuLoader.addU   s    

4+&r6   c                      y rA   r5   rG   s    r7   rH   z_SerialCpuLoader.start_loadingX       r6   c              #     K   | j                   D ]y  \  }}| j                  |      j                         }|j                         }|j	                         j                         |j                         k7  r|j                         }||f { y wrA   )rR   rP   detachcpustorager=   numelclonerC   _r>   tensors       r7   rJ   z_SerialCpuLoader.values[   sz     jj 	FAs%%c*113FZZ\F~~$$&&,,.8 	s   B
BrK   )r.   r/   r0   r   rS   r4   rL   rD   rH   r   r   rM   r   rJ   r5   r6   r7   rO   rO   P   sS    2H 2 2' '& 'T '	u||V';!<= 	r6   rO   c            	          e Zd Z	 	 ddedeej                     deddfdZe	de
fd       Zdeeej                  ef      fdZdd	Zdeeej                  ef      fd
ZdededdfdZddZdeeej                  ef      fdZy)_OverlappingCpuLoaderNrP   streaminflight_threshholdr?   c                 8   || _         g | _        || _        d| _        t	        j
                         | _        d| _        d| _        |r|j                  n	t               | _	        t        | j                        | _        t        t        j                  j                   |xs | j                  j#                               | _        | j$                  | j                  j#                         k7  r4| j$                  j'                  | j                  j#                                y y )Nr   F)rP   rR   rd   in_flight_datacollectionsdequecurrent_itemsidxstarteddevice_typer   r   device_moduler	   rM   cudaStreamcurrent_streamrc   wait_stream)rC   rP   rc   rd   s       r7   rS   z_OverlappingCpuLoader.__init__h   s     '/1
#6 0;0A0A0C"(F.H.J 	 00@0@AJJvL););)J)J)L
 ;;$,,;;==KK##D$6$6$E$E$GH >r6   c                 F    | j                   t        | j                        k\  S rA   )rj   lenrR   rG   s    r7   _donez_OverlappingCpuLoader._done   s    xx3tzz?**r6   c                    g }| j                   | j                  k\  r| j                  j                          | j                   | j                  k\  r| j                  j                         }| xj                   |d   j                         |d   j                         z  z  c_         |j                  |       | j                   | j                  k\  r|S Nr   )	rf   rd   rc   synchronizeri   popleftr\   element_sizerU   )rC   drainedvals      r7   _drainz_OverlappingCpuLoader._drain   s    $":"::KK##%!!T%=%==$$,,.C3q6<<>CF4G4G4I#IINN3 !!T%=%== r6   c                    | j                   j                  | j                        5  | j                  s| j                  | j                  k  rm| j
                  | j                     \  }}| xj                  dz  c_        | j                  |      j                         }|j                  j                  | j                  k(  r|j                  dd      }nn|j                  t        j                  d      k(  rL|j                         j                         |j!                         |j"                  z  k7  r|j%                         }| j&                  j)                  ||f       | xj                  |j!                         |j+                         z  z  c_        | j                  s| j                  | j                  k  rmd d d        y # 1 sw Y   y xY w)Nr   rZ   T)devicenon_blocking)rm   rc   rt   rf   rd   rR   rj   rP   rY   r~   typerl   torM   untyped_storager=   r\   itemsizer]   ri   rU   ry   r^   s       r7   _refillz_OverlappingCpuLoader._refill   sc   &&t{{3 	NjjT%8%84;S;S%SDHH-3A))#.557==%%)9)99#YYe$YGF]]ell5&99..0557!<<>FOO;< "("")) ##v||~8K8K8M'MM#) jjT%8%84;S;S%S	N 	N 	Ns   FGGc                     | j                   sJ t        | j                        dkD  r| j                  j	                          | j                  S rv   )rt   rs   ri   rc   rw   rG   s    r7   _finishz_OverlappingCpuLoader._finish   s=    zzzt!!"Q&KK##%!!!r6   r=   r>   c                 l    | j                   rt        d      | j                  j                  ||f       y )Nz&cannot add items after loading started)rk   RuntimeErrorrR   rU   rB   s      r7   rD   z_OverlappingCpuLoader.add   s+    <<GHH

4+&r6   c                     | j                   ry d| _         | j                  j                  d        | j                          y )NTc                     | d   S rv   r5   xs    r7   <lambda>z5_OverlappingCpuLoader.start_loading.<locals>.<lambda>   s
    ad r6   key)rk   rR   sortr   rG   s    r7   rH   z#_OverlappingCpuLoader.start_loading   s.    <<

N+r6   c              #      K   | j                          | j                  s7| j                         }| j                          |E d {    | j                  s7| j	                         E d {    y 7 *7 wrA   )rH   rt   r|   r   r   )rC   rz   s     r7   rJ   z_OverlappingCpuLoader.values   sY     **kkmGLLN **
 <<>!! !s*   AA3A/A3A3)A1*A31A3)Ni@B rK   )r.   r/   r0   r   r   rM   ro   r4   rS   propertyboolrt   r   r   r   rL   r|   r   r   r   rD   rH   r   rJ   r5   r6   r7   rb   rb   g   s     *.#,	II &I !	I
 
I. +t + +U5<<#789 N0"%f(<"=> "' '& 'T '
"u||V';!<= "r6   rb   itemr?   c                     d}| j                   J | j                   j                  D ]  }||z  }	 | j                   j                  j                  }|t        j
                  j                  |      z  S Nr   )tensor_datar=   
propertiesdtyperM   _utils_element_size)r   r=   sr   s       r7   
_item_sizer      sp    D'''"" 	 ''--E%,,,,U333r6   binsrR   c                 j   | dk(  r|gS |D cg c]"  }|j                   t        j                  k(  s!|$ }}|D cg c]"  }|j                   t        j                  k7  s!|$ }}t        |       D cg c]  }g  }}t        |       D cg c]  }d }}|j	                  t
        d       t        |      D ]  \  }}||| z     j                  |        |D ]F  }t        t        |      d       d   }	||	   j                  |       ||	xx   t        |      z  cc<   H |S c c}w c c}w c c}w c c}w )Nr   r   T)r   reversec                     | d   S r   r5   r   s    r7   r   z)_split_by_size_and_type.<locals>.<lambda>   s
    1 r6   r   )	r   r"   BYTE_IOranger   r   	enumeraterU   min)
r   rR   wibytes_wtensor_wr_   bucketsbucket_sizesirj   s
             r7   _split_by_size_and_typer      s(   qyw!FbRWW0E0E%ErFGF"Grbgg1F1F&FGHG27+%>Qb%>G%>$T{+!A+L+MMj$M/7# %2D  $%  ,)L)~>qABSZ^+	, N# GG%>+s!   "D!D!"D&D&0	D+	D0rc   data
write_itemstorage_keyc           	         | j                         }|j                  t        j                  k(  r<t	        |t
        j                        sJ | j                  |j                                nkt	        |t        j                        sJ |j                  t        j                  d      k(  sJ t        j                  |t        t        t           |              | j                         |z
  }t!        |j"                  |t%        |||            S )NrZ   )indexsize_in_bytesstorage_data)tellr   r"   r   
isinstanceioBytesIOwrite	getbufferrM   r   r~   saver	   r   bytesr%   r   r*   )rc   r   r   r   r,   r-   s         r7   _write_itemr      s     [[]F-///$

+++T^^%&$---{{ell51111

4bi01[[]V#F!+vv> r6   create_stream
file_queueresult_queueplannerrd   	use_fsyncthread_countc           
         	 	 |j                         \  }}}	t        j                  j                         }
t	        t        |
d       }|dk(  rMt        j
                  j                         s|r-|j                         r|dkD  rt        |j                  |      }nt        |j                        }|	D cg c]"  }|j                  t        j                  k7  s!|$ }}|D ]  }|j                  t        |      |        |j                          |	D cg c]"  }|j                  t        j                  k(  s!|$ }}g } | |d      5 }|D ]0  }|j                  |      }|j!                  t#        ||||             2 |j%                         D ]0  \  }}|j&                  sJ |j!                  t#        ||||             2 |r$	 t)        j*                  |j-                                d d d        |j3                  |       c c}w c c}w # t.        $ r t)        j0                          Y Dw xY w# 1 sw Y   IxY w# t4        j6                  $ r Y y w xY w)Nr   r   )rd   wb)
get_nowaitrM   _C_get_privateuse1_backend_namegetattrrn   is_availablerb   resolve_datarO   r   r"   r   rD   r   rH   rU   r   rJ   is_cpuosfsyncfilenoAttributeErrorsyncputqueueEmpty)r   r   r   r   rd   r   r   	file_namer   write_itemscustom_backend_namecustom_device_modloaderr   r   r   r   write_resultsrc   r   r`   s                        r7   _write_files_from_queuer      s,   82<2G2G2I/I{K #((("H"H"J '/BD I !JJ++-).?.L.L.N'!+.(((;
 *(( &1UrBGG}?T?T4TUHU& ?


:j1:>?  "$/Tb277m>S>S3SrTGTMy$/ "6") J"//
;D!((#FD*kJ +1--/ &FJ!==(=!((#FFJL "1"$ ]+k 6 V
 U& * "	"!" "& ;; sm   B/H7 1"G>G>8H7 "H3H7H7 A;H+ #H#%H7 H(%H+'H((H++H40H7 7IIc                      e Zd Zeedeeej                  f   dede	e
j                  ddf   fd              Zedeeej                  f   dedeeej                  f   fd       Zedeeej                  f   deeej                  f   ddfd	       Zedeeej                  f   deeej                  f   fd
       Zedeeej                  f   ddfd       Zeedeeej                  f   defd              Zy)FileSystemBasepathmoder?   Nc                      y rA   r5   )rC   r   r   s      r7   r   zFileSystemBase.create_streamA  s    
 	r6   suffixc                      y rA   r5   rC   r   r   s      r7   concat_pathzFileSystemBase.concat_pathH       	r6   new_pathc                      y rA   r5   rC   r   r   s      r7   renamezFileSystemBase.renameN  r   r6   c                      y rA   r5   rC   r   s     r7   	init_pathzFileSystemBase.init_pathT      r6   c                      y rA   r5   r   s     r7   mkdirzFileSystemBase.mkdirX  r   r6   checkpoint_idc                      y rA   r5   clsr   s     r7   validate_checkpoint_idz%FileSystemBase.validate_checkpoint_id\  s     	r6   )r.   r/   r0   r   r   r   r2   r   PathLiker   r   IOBaser   r   r   r   r   classmethodr   r   r5   r6   r7   r   r   @  s   #r{{*+36	299dD(	)  
 #r{{*+58	sBKK	  
 #r{{*+7<S"++=M7N	 
 eC$45 %R[[@P:Q   %R[[ 01 d   5bkk9I3J t   r6   r   c            
          e Zd Zedeeej                  f   dedee	j                  ddf   fd       Zdeeej                  f   dedeeej                  f   fdZdeeej                  f   deeej                  f   fdZdeeej                  f   d	eeej                  f   ddfd
Zdeeej                  f   ddfdZedeeej                  f   defd       Zy)
FileSystemr   r   r?   Nc              #      K   t        t        |      j                  |      5 }t        t        j                  |       d d d        y # 1 sw Y   y xY wwrA   )r	   r   openr   r   )rC   r   r   rc   s       r7   r   zFileSystem.create_streamc  sD      $""4( 	*Fryy&))	* 	* 	*s    AA	AAAr   c                 (    t        t        |      |z  S rA   )r	   r   r   s      r7   r   zFileSystem.concat_pathj  s     D$&((r6   c                 <    t        |t              st        |      }|S rA   )r   r   r   s     r7   r   zFileSystem.init_patho  s    $%:Dr6   r   c                 ^    t        t        |      j                  t        t        |             y rA   )r	   r   r   r   s      r7   r   zFileSystem.renamet  s      	T4T8 45r6   c                 F    t        t        |      j                  dd       y )NT)parentsexist_ok)r	   r   r   r   s     r7   r   zFileSystem.mkdiry  s    T4td;r6   r   c                     t        |t              rydt        |      v ryt        |      j                  D ]B  }|j	                         st        j                  t        |      t
        j                        sB y y)NTz://F)r   r   r2   r   existsr   accessW_OK)r   r   ps      r7   r   z!FileSystem.validate_checkpoint_id|  s_    mT*C&&m$,, 	AxxzbiiA8	 r6   )r.   r/   r0   r   r   r2   r   r   r   r   r   r   r   r   r   r   r   r   r   r5   r6   r7   r   r   b  s<   *#r{{*+*36*	299dD(	)* *)#r{{*+)58)	sBKK	 )
eC$45 %R[[@P:Q 
6#r{{*+67<S"++=M7N6	6
<%R[[ 01 <d < 5bkk9I3J t  r6   r   c                   J    e Zd ZdZ	 	 	 	 ddeeej                  f   dedede	de	ddf fd	Z
dd
eeej                  df   ddfdZdeddfdZdedefdZdee   dee   fdZdededeee      fdZdedeee      ddfdZed
eeej                  f   defd       Z xZS )r'   aa  
    Basic implementation of StorageWriter using file IO.

    This implementation makes the following assumptions and simplifications:

    * The checkpoint path is an empty or non-existing directory.
    * File creation is atomic

    The checkpoint consist of one file per write request plus
    a `.metadata` file with the serialized metadata.

    r   single_file_per_rank
sync_filesr   per_thread_copy_aheadr?   Nc                     t         |           t               | _        | j                  j	                  |      | _        || _        || _        || _        || _	        y)a  
        Initialize the writer pointing to `path`.

        Args:
            path: directory where the checkpoint will be written to.
            single_file_per_rank: Produce one file per rank instead of one file per tensor/blob. Default to True.
            sync_files : force files to be synced to permanent storage. Default to True.
            thread_count: Number of IO threads to use to write. Default to 1.
            per_thread_copy_ahead: How many bytes to copy from the GPU ahead of saving then. Default 10Mb.

        N. B. If sync_files is disabled, there's no guarantee that the checkpoint will be consistent in the case of a failure.
        N)
superrS   r   fsr   r   r  r  r   r	  )rC   r   r  r  r   r	  	__class__s         r7   rS   zFileSystemWriter.__init__  sN    ( 	,GG%%d+	$8!$(%:"r6   r   c                 J    |r!| j                   j                  |      | _        y y rA   )r  r   r   rC   r   s     r7   resetzFileSystemWriter.reset  s     ))-8DI r6   is_coordinatorc                      y rA   r5   )rC   r  s     r7   set_up_storage_writerz&FileSystemWriter.set_up_storage_writer  rW   r6   planc                 P    | j                   j                  | j                         |S rA   )r  r   r   rC   r  s     r7   prepare_local_planz#FileSystemWriter.prepare_local_plan  s    dii r6   global_planc                     t        |      D cg c])  \  }}t        j                  |t        d| d            + }}}|S c c}}w )N__r_   r   )r   dataclassesreplacer9   )rC   r  r   r  	new_planss        r7   prepare_global_planz$FileSystemWriter.prepare_global_plan  sQ     %[1
4 >Bqc)3LM
	 
 	
s   .Ar   c                    |j                   dfd}t        j                         }| j                  rgt	        | j
                  |j                        D ]C  } |       }| j                  j                  | j                  |      }|j                  |||f       E nS|j                  D ]D  } |       }| j                  j                  | j                  |      }|j                  |||gf       F t        j                         }	g }
t        d| j
                        D ]w  }t        j                  t        | j                  j                  ||	|| j                   | j"                  | j
                  f      }|j%                          |
j'                  |       y t        | j                  j                  ||	|| j                   | j"                  | j
                         |
D ]  }|j)                           g }	 	 ||	j+                         z  }# t        j,                  $ r! 	 t/               }|j1                  |       |cY S w xY w)Nr   c                  >    j                     t         } dz  | S r   )r:   DEFAULT_SUFFIX)r   
file_countstorage_plans    r7   gen_filez-FileSystemWriter.write_data.<locals>.gen_file  s,    '../
|N;KLI!OJr6   r   )targetargs)r   r   r   r   rd   r   r   )r   r   Queuer  r   r   rR   r  r   r   r   r   	threadingThreadr   r   r	  r  startrU   joinr   r   r   
set_result)rC   r  r   r%  r   bucketr   r   r   r   threadsr_   tresfutr#  r$  s                  @@r7   
write_datazFileSystemWriter.write_data  s   
 (,'8'8
	 #(++-
$$1$2C2CTZZP :$J	ww**499i@i89:
 

 :$J	ww**499i@i$89:
 %*KKMq$++, 	A  .GG)) ..OO%%A GGINN1	  	 ''//!% $ : :oo**	
  	AFFH	 	|..00 {{ 	-3XCNN3J	s   5H
 
1H>=H>metadataresultsc                    t               }|D ]6  }|j                  |D ci c]  }|j                  |j                   c}       8 ||_        t	        t
        | j                  j                  | j                  d            }t	        t
        | j                  j                  | j                  d            }| j                  j                  |d      5 }t        j                  ||       | j                  r$	 t        j                  |j                                d d d        | j                  j%                  ||       y c c}w # t         $ r t        j"                          Y Iw xY w# 1 sw Y   NxY w)Nz.metadata.tmp	.metadatar   )dictupdater   r   r	   r   r  r   r   r   pickledumpr  r   r   r   r   r   r   )	rC   r4  r5  
storage_mdwr_listwrtmp_path	meta_pathmetadata_files	            r7   finishzFileSystemWriter.finish  s   V
 	MG7KRrxx8KL	M *dgg11$))_MNtww22499kJK	WW""8T2 	mKK-0HH]1134		 	x+ L & GGI	 	s/   D=
#E%5#EE"E%!E""E%%E.c                 ,    t         j                  |      S rA   r   r   r   s     r7   r   z'FileSystemWriter.validate_checkpoint_id      00??r6   )TTr   i rA   )r.   r/   r0   r1   r   r2   r   r   r   r4   rS   r  r  r   r  r   r  r    r   r%   r3  r   rB  r   r   __classcell__r  s   @r7   r'   r'     sO     &*%/;C$%; #; 	;
 ;  #; 
;895bkk4)?#@ 9D 9D T x H tH~ $x. CC C 
[!	"	CJ,x ,$tK7H2I ,d ," @5bkk9I3J @t @ @r6   c                   :    e Zd Zdeeej                  f   ddf fdZdede	j                  fdZddeeej                  df   ddfdZd	ed
eded   fdZdefdZdededdfdZd	edefdZdee   dee   fdZedeeej                  f   defd       Z xZS )r(   r   r?   Nc                     t         |           t               | _        | j                  j	                  |      | _        t               | _        y rA   )r  rS   r   r  r   r   r8  r   )rC   r   r  s     r7   rS   zFileSystemReader.__init__#  s7    ,GG%%d+	?Cvr6   sinfoc                 D    t        ||j                  |j                        S rA   )r&   r,   r-   )rC   filerJ  s      r7   _slice_filezFileSystemReader._slice_file)  s     u||U\\BBr6   r   c                 h    t               | _        |r!| j                  j                  |      | _        y y rA   )r8  r   r  r   r   r  s     r7   r  zFileSystemReader.reset,  s*     F))-8DI r6   r  r   c                    t               }|j                  D ]H  }| j                  |j                     }|j                  }|j                  |g       j                  |       J |j                         D ]  \  }}| j                  j                  | j                  |      }	| j                  j                  |	d      5 }
|D ]  }| j                  |j                     }| j                  |
|      }|j                  t        j                  k(  rRt        j                   |j#                  |j$                              }|j'                  d       |j)                  ||       t+        t,        t/        j0                  t+        t2        t4           |      d            }t7        ||j8                  |j:                        }|j=                  |      j?                         }|jA                         |jA                         k(  s6J d|j                   d|jA                          d|jA                                 |jC                  |       |jE                  ||        	 d d d         tG               }|jI                  d        |S # 1 sw Y   xY w)Nrbr   rZ   )map_locationzreq z mismatch sizes z vs )%r8  rR   r   storage_indexr+   
setdefaultrU   r  r   r   r   rM  r   r   r   r   r   readr-   seek
load_bytesr	   r   rM   loadr   r   r   storage_offsetslengthsresolve_tensorrY   r=   copy_commit_tensorr   r-  )rC   r  r   per_file	read_itemitem_mdr   r+   reqsr   rc   req
file_slice
read_bytesr`   target_tensorr2  s                    r7   	read_datazFileSystemReader.read_data1  s;   .2f 	<I''	(?(?@G((Db)00;	<
 $,>>#3 	BM4ww**499mDH&&x6 B& BC"//0A0ABG!%!1!1&'!BJxx<#7#77%'ZZ
0O%P
"***3
;!%"!JJtBuIz'BQVW" "8"C$7$7" )0(>(>s(C(J(J(L *..0FKKMAo!#"3"3!44D]EWEWEYDZZ^_e_j_j_l^mnoA%++F3--c=A+BB B	B6 ht
7B Bs   ;FI<<J	c                     | j                   j                  | j                  d      }| j                   j                  |d      5 }t	        j
                  |      cd d d        S # 1 sw Y   y xY w)Nr7  rP  )r  r   r   r   r:  rW  )rC   r   rA  s      r7   read_metadatazFileSystemReader.read_metadataY  sS    ww""499k:WW""4. 	.-;;}-	. 	. 	.s   A""A+r4  r  c                 B    |j                   | _         | j                   J y rA   r  )rC   r4  r  s      r7   set_up_storage_readerz&FileSystemReader.set_up_storage_reader^  s"    $11  ,,,r6   c                     |S rA   r5   r  s     r7   r  z#FileSystemReader.prepare_local_planb  s    r6   r  c                     |S rA   r5   )rC   r  s     r7   r  z$FileSystemReader.prepare_global_plane  s    r6   c                 ,    t         j                  |      S rA   rD  r   s     r7   r   z'FileSystemReader.validate_checkpoint_idh  rE  r6   rA   )r.   r/   r0   r   r2   r   r   rS   r*   r   r   rM  r  r   r   r   re  r   rg  r   ri  r  r   r  r   r   rF  rG  s   @r7   r(   r(   "  s   FU3#34 F FC| C		 C95bkk4)?#@ 9D 9
%h % % %P.x .
-h - -QU -x H tH~ $x.  @5bkk9I3J @t @ @r6   )Krg   r  r   r   r:  r   r)  abcr   r   
contextlibr   r   pathlibr   typingr   r	   r
   r   r   r   r   r   r   r   r   rM   r   torch._utilsr   r   torch.distributed._shard._utilsr   torch.futuresr   r4  r   r   r   r   r   r   r   r   r    r!   r"   r[   r#   r$   r%   utilsr&   __all__r*   r9   r"  r<   rO   rb   r4   r   r   r   r   r2   r   r(  r   r   r   r   r'   r(   r5   r6   r7   <module>rv     s     	 	    # % !       G B   -	 	 	 ? > $1
2       C } .W"M W"t4Y 43 4# d9o $tIBW 0II


ELL(
)  	
 0AAA ++A 	A
 A A A 
AHS D& &RT@} T@nH@} H@r6   