
    '}h                         d dl Z d dlZd dlmZ d dlmZ d dlmZmZm	Z	 d dl
Z
d dl
mZ d dlmZ d dlmZmZmZ dd	gZ G d
 de      Z G d de      Z G d d	e      Zy)    N)contextmanager)Path)	GeneratorOptionalUnion)AbstractFileSystem)	url_to_fs)FileSystemBaseFileSystemReaderFileSystemWriterFsspecWriterFsspecReaderc            
          e Zd ZddZedeeej                  f   dede	e
j                  ddf   fd       Zdeeej                  f   dedeeej                  f   fdZdeeej                  f   deeej                  f   fd	Zdeeej                  f   d
eeej                  f   ddfdZdeej                  gddfdZedeeej                  f   defd       Zy)
FileSystemreturnNc                     d | _         y N)fs)selfs    b/var/www/html/test/engine/venv/lib/python3.12/site-packages/torch/distributed/checkpoint/fsspec.py__init__zFileSystem.__init__   s	    04    pathmodec              #      K   | j                   J | j                   j                  5  t        j                  t	        |      |      5 }| d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY wwr   )r   transactionfsspecopenstr)r   r   r   streams       r   create_streamzFileSystem.create_stream   sl      ww"""WW   	SY- 	 	 	 	s4   %A5 A)AA)	A5A&	"A))A2.A5suffixc                 B    t         j                  j                  ||      S r   )osr   join)r   r   r"   s      r   concat_pathzFileSystem.concat_path'   s     ww||D&))r   c                 ,    t        |      \  | _        }|S r   )r	   r   )r   r   _s      r   	init_pathzFileSystem.init_path,   s    t_
r   new_pathc                 <    | j                   j                  ||       y r   )r   rename)r   r   r*   s      r   r,   zFileSystem.rename0   s     	tX&r   c                 >    | j                   j                  |d       y )NT)exist_ok)r   makedirs)r   r   s     r   mkdirzFileSystem.mkdir5   s    -r   checkpoint_idc                 j    t        |t              ry	 t        |       y# t        $ r
}Y d }~yd }~ww xY w)NFT)
isinstancer   r	   
ValueError)clsr1   es      r   validate_checkpoint_idz!FileSystem.validate_checkpoint_id8   s6    mT*	m$   		s    	22)r   N)__name__
__module____qualname__r   r   r   r   r$   PathLiker   ioIOBaser!   r&   r)   r,   r0   classmethodboolr7    r   r   r   r      s>   5 #r{{*+36	299dD(	) *#r{{*+*58*	sBKK	 *
eC$45 %R[[@P:Q '#r{{*+'7<S"++=M7N'	'
.3, . . 	5bkk9I3J 	t 	 	r   r   c                        e Zd ZdZ	 	 	 	 ddeeej                  f   dedede	de	ddf fd	Z
ed
eeej                  f   defd       Z xZS )r   a`  
    Basic implementation of StorageWriter using FFspec.

    This implementation makes the following assumptions and simplifications:

    * The checkpoint path is an empty or non-existing directory.
    * File creation is atomic

    The checkpoint consist of one file per write request plus
    a `.metadata` file with the serialized metadata.

    r   single_file_per_rank
sync_filesthread_countper_thread_copy_aheadr   Nc                     t         |   |||||       t               | _        | j                  j	                  |      | _        y)a  
        Initialize the writer pointing to `path`.

        Args:
            path: directory where the checkpoint will be written to.
            single_file_per_rank: Produce one file per rank instead of one file per tensor/blob. Default to True.
            sync_files : force files to be synced to permanent storage. Default to True.
            thread_count: Number of IO threads to use to write. Default to 1.
            per_thread_copy_ahead: How many bytes to copy from the GPU ahead of saving then. Default 10Mb.

        N. B. If sync_files is disabled, there's no guarantee that the checkpoint will be consistent in the case of a failure.
        Nsuperr   r   r   r)   r   )r   r   rB   rC   rD   rE   	__class__s         r   r   zFsspecWriter.__init__S   s@    ( 	&
LBW	
 ,GG%%d+	r   r1   c                 ,    t         j                  |      S r   )r   r7   r5   r1   s     r   r7   z#FsspecWriter.validate_checkpoint_idm   s    00??r   )TT   i )r8   r9   r:   __doc__r   r   r$   r;   r?   intr   r>   r7   __classcell__rI   s   @r   r   r   E   s      &*%/,C$%, #, 	,
 ,  #, 
,4 @5bkk9I3J @t @ @r   c                   ~     e Zd Zdeeej                  f   ddf fdZedeeej                  f   de	fd       Z
 xZS )r   r   r   Nc                     t         |   |       t               | _        | j                  j	                  |      | _        y r   rG   )r   r   rI   s     r   r   zFsspecReader.__init__s   s/    ,GG%%d+	r   r1   c                 ,    t         j                  |      S r   )r   checkrK   s     r   r7   z#FsspecReader.validate_checkpoint_idx   s    ..r   )r8   r9   r:   r   r   r$   r;   r   r>   r?   r7   rO   rP   s   @r   r   r   r   sT    ,U3#34 , ,
 /5bkk9I3J /t / /r   )r<   r$   
contextlibr   pathlibr   typingr   r   r   r   r   fsspec.corer	   'torch.distributed.checkpoint.filesystemr
   r   r   __all__r   r   r   r@   r   r   <module>r[      se    
 	 %  - -  % !  ( (V*@# *@Z/# /r   