|hLddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z m Z m Z mZmZmZddlZddlZddlmZddlmZmZmZmZddlmZmZmZmZm Z dd l!m"Z"Gd d eZ#y) N)deepcopy) ThreadPool)Path)AnyDictListOptionalTupleUnion)Dataset)FORMATS_HELP_MSGHELP_URL IMG_FORMATScheck_file_speeds) DEFAULT_CFG LOCAL_RANKLOGGER NUM_THREADSTQDM)imreadceZdZdZdddeddddddd d d f d eeeefd edee efde de ee fdede dedede de de eede deffd Zd eeeefdeefdZde eedd fdZd0ded e deej&eeefeeefffd!Zd1d"Zdedd fd#Zd2d$e de fd%Zd2d$e de fd&Zd1d'Zd(ede ee ffd)Zd(ede ee ffd*Zdefd+Zd,e ee fde ee ffd-Zd3de e ee ffd.Zdee ee ffd/ZxZ S)4 BaseDataseta Base dataset class for loading and processing image data. This class provides core functionality for loading images, caching, and preparing data for training and inference in object detection tasks. Attributes: img_path (str): Path to the folder containing images. imgsz (int): Target image size for resizing. augment (bool): Whether to apply data augmentation. single_cls (bool): Whether to treat all objects as a single class. prefix (str): Prefix to print in log messages. fraction (float): Fraction of dataset to utilize. channels (int): Number of channels in the images (1 for grayscale, 3 for RGB). cv2_flag (int): OpenCV flag for reading images. im_files (List[str]): List of image file paths. labels (List[Dict]): List of label data dictionaries. ni (int): Number of images in the dataset. rect (bool): Whether to use rectangular training. batch_size (int): Size of batches. stride (int): Stride used in the model. pad (float): Padding value. buffer (list): Buffer for mosaic images. max_buffer_length (int): Maximum buffer size. ims (list): List of loaded images. im_hw0 (list): List of original image dimensions (h, w). im_hw (list): List of resized image dimensions (h, w). npy_files (List[Path]): List of numpy file paths. cache (str): Cache images to RAM or disk during training. transforms (callable): Image transformation function. batch_shapes (np.ndarray): Batch shapes for rectangular training. batch (np.ndarray): Batch index of each image. Methods: get_img_files: Read image files from the specified path. update_labels: Update labels to include only specified classes. load_image: Load an image from the dataset. cache_images: Cache images to memory or disk. cache_images_to_disk: Save an image as an *.npy file for faster loading. check_cache_disk: Check image caching requirements vs available disk space. check_cache_ram: Check image caching requirements vs available memory. set_rectangle: Set the shape of bounding boxes as rectangles. get_image_and_label: Get and return label information from the dataset. update_labels_info: Custom label format method to be implemented by subclasses. build_transforms: Build transformation pipeline to be implemented by subclasses. get_labels: Get labels method to be implemented by subclasses. iFT ?Ng?img_pathimgszcacheaugmenthypprefixrect batch_sizestridepad single_clsclassesfractionchannelsc$t|||_||_||_| |_||_| |_||_|dk(rtjntj|_ |j|j|_|j|_|j#| t%|j |_||_||_| |_| |_|j(r|j*J|j1g|_|jr%t5|j&|j*dzdfnd|_dg|j&zdg|j&zdg|j&zc|_|_|_|jDcgc]}t?|jAdc}|_!tE|tFr|jIn|durd nd|_%|jJd k(rB|jMr2|jNrtQjRd |jUn/|jJd k(r |jWr|jU|jY| |_-ycc}w) a Initialize BaseDataset with given configuration and options. Args: img_path (str | List[str]): Path to the folder containing images or list of image paths. imgsz (int): Image size for resizing. cache (bool | str): Cache images to RAM or disk during training. augment (bool): If True, data augmentation is applied. hyp (Dict[str, Any]): Hyperparameters to apply data augmentation. prefix (str): Prefix to print in log messages. rect (bool): If True, rectangular training is used. batch_size (int): Size of batches. stride (int): Stride used in the model. pad (float): Padding value. single_cls (bool): If True, single class training is used. classes (List[int], optional): List of included classes. fraction (float): Fraction of dataset to utilize. channels (int): Number of channels in the images (1 for grayscale, 3 for RGB). ) include_classNirz.npyTramzcache='ram' may produce non-deterministic training results. Consider cache='disk' as a deterministic alternative if your disk space allows.disk)r").super__init__rrr!r(r#r*r+cv2IMREAD_GRAYSCALE IMREAD_COLORcv2_flag get_img_filesim_files get_labelslabels update_labelslennir$r%r&r' set_rectanglebufferminmax_buffer_lengthimsim_hw0im_hwr with_suffix npy_files isinstancestrlowerr check_cache_ram deterministicrwarning cache_imagescheck_cache_diskbuild_transforms transforms)selfrrr r!r"r#r$r%r&r'r(r)r*r+f __class__s T/var/www/html/test/engine/venv/lib/python3.12/site-packages/ultralytics/data/base.pyr3zBaseDataset.__init__FsH     $     08A ,,3CSCS **4==9 oo'  1dkk" $  99??. ..     NRlldggt/BD%I!J`a.2FTWW,NQUPVY]Y`Y`P`)$+tz?C}}M!$q'--f5M&0&z,BaseDataset.get_img_files..s@p1<*5 KKN;7 //44Q7(+A Au%+1!9 Ax(NWXYl1b72s^a(2,1bDKKN:.(2;A,DKKN;/./ Au%ad+ 02cs  E*Er rect_modec^|j||j||j|}}}|b|jr t j |}nt||j}|td||jdd\}}|r|j t#||z } | d k7rt%t'j(|| z|j t%t'j(|| z|j } } t+j,|| | ft*j. }nS||cxk(r|j k(s>nt+j,||j |j ft*j. }|j0dk(r|d }|j2r|||f|jddc|j|<|j4|<|j6|<|j8j;|d t=|j8cxkr|j>k\rZnnW|j8jAd } |jBd k7r-d\|j| <|j4| <|j6| <|||f|jddfS|j||j4||j6|fS#t $rd}tj|jd|d|t|jdt||j}Yd}~d}~wwxYw)a Load an image from dataset index 'i'. Args: i (int): Index of the image to load. rect_mode (bool): Whether to use rectangular resizing. Returns: im (np.ndarray): Loaded image as a NumPy array. hw_original (Tuple[int, int]): Original image dimensions in (height, width) format. hw_resized (Tuple[int, int]): Resized image dimensions in (height, width) format. Raises: FileNotFoundError: If the image file is not found. Nz"Removing corrupt *.npy image file z due to: T) missing_ok)flagszImage Not Found r-) interpolation).Nrr0)NNN)"rCr9rGexistsr|loadrrrrMr#runlinkrr7rpshapermaxrAmathceilr4resize INTER_LINEARndimr!rDrEr@appendr=rBpopr ) rRrrimrSfnrvh0w0rwhrs rU load_imagezBaseDataset.load_images HHQKq!14>>!3DrA :yy{8B AT]]3z'*:1#(>??XXbq\FBJJR,6 "q& 14::>DIIbSTfDUW[WaWa@bqABAc>N>NOBB,$**,ZZTZZ$rrrrGstatst_sizerCrDrEnbytesr#descclose) rRbgbfcnstoragepoolresultspbarrrcs rUrNzBaseDataset.cache_imagess<2>BjjF>R116:Y]YhYhjoXp W  $ iiU477^4G '*$'':PQ>RD V1::'*//1999AAB>DHHQKQA!+++A#{{m+;AF3rrandomchoicer9rrr`accessrrnW_OKr rrMr# disk_usageint) rRrrrrn_im_filer disk_requiredrusedfrees rUrOzBaseDataset.check_cache_disksZ 2  q AmmDMM2GBz NA99T']11277;! $++.fgh DGG a1}+<= "--d4==3C.D.K.KLtT 4 DJ NN;;-  2378MC/011J"9S/52:c"22UW  rc d\}}t|jd}t|D]u}tt j |j }|.|jt|jd|jdz }||j|dzzz }w||jz|z d|zz}tdj} || jkDrad|_tj |j"||z dd t%|d zd | j|z dd | j&|z dd yy)z Check if there's enough RAM for caching images. Args: safety_margin (float): Safety margin factor for RAM calculation. Returns: (bool): True if there's enough RAM, False otherwise. rrNrr-rpsutilrz%GB RAM required to cache images with rrr]z GB available, not caching imagesFT)rAr>rrrrr9rrrr __import__virtual_memory availabler rrMr#rr) rRrrrrrrratio mem_requiredmems rUrKzBaseDataset.check_cache_ramBs:2  q &A dmm45BzJJRXXa["((1+!>>E UAX% %A  & 477{Q!m*;< "113 #-- 'DJ NN;;- r 1#67MC/011J==2%c*!CIIN3+??_a  rctjtj|j|jz j t }|ddz}tj|jDcgc]}|jdc}}|dddf|dddfz }|j}|Dcgc]}|j|c}|_ |Dcgc]}|j|c}|_||}ddgg|z}t|D]G}|||k(} | j| j} } | dkr| dg||<8| dkDs>dd| z g||<Itjtj||j z|j"z |j$zj t |j"z|_||_ycc}wcc}wcc}w)zBSet the shape of bounding boxes for YOLO detections as rectangles.r\r-rNr)r|flooraranger>r%astyperr}r;rargsortr9rrArrrr&r' batch_shapesbatch) rRbinbrcsarirectrshapesariminimaxis rUr?zBaseDataset.set_rectangle`s XXbii(4??: ; B B3 G VaZ HHdkk:aeeGn: ; q!tWqAw  389aq)9 /45!t{{1~5 Ya&Br *AR1W+CCGGI$Dax!1Iq DMq  *GGBHHV$4tzz$ADKK$ORVRZRZ$Z[bbcfgjnjujuu %;:5s8G>G! G&indexcB|j|j|S)z5Return transformed label information for given index.)rQget_image_and_label)rRrs rU __getitem__zBaseDataset.__getitem__yst77>??rcTt|j|}|jdd|j|\|d<|d<|d<|dd|ddz |dd|ddz f|d<|jr|j |j ||d <|j|S) z Get and return label information from the dataset. Args: index (int): Index of the image to retrieve. Returns: (Dict[str, Any]): Label dictionary with image and metadata. rNimg ori_shape resized_shaperr- ratio_pad rect_shape)rr;rrr$rrupdate_labels_info)rRrlabels rUrzBaseDataset.get_image_and_label}sU+, '4 CG??SXCY@e eK(%*@ / "1 %k(:1(= = / "1 %k(:1(= = k 99"&"3"3DJJu4E"FE, &&u--rc,t|jS)z5Return the length of the labels list for the dataset.)r=r;rRs rU__len__zBaseDataset.__len__s4;;rrc|S)zCustom your label format here.)rRrs rUrzBaseDataset.update_labels_infos rct)a Users can customize augmentations here. Examples: >>> if self.augment: ... # Training transforms ... return Compose([]) >>> else: ... # Val transforms ... return Compose([]) NotImplementedError)rRr"s rUrPzBaseDataset.build_transformss "!rct)a) Users can customize their own format here. Examples: Ensure output is a dictionary with the following keys: >>> dict( ... im_file=im_file, ... shape=shape, # format: (height, width) ... cls=cls, ... bboxes=bboxes, # xywh ... segments=segments, # xy ... keypoints=keypoints, # xy ... normalized=True, # or False ... bbox_format="xyxy", # or xywh, ltwh ... ) rrs rUr:zBaseDataset.get_labelss ""!r)T)rVN)r)N)!__name__ __module__ __qualname____doc__rr rIrrboolrrfloatr r3r8r<r r|ndarrayrrNrrOrKr?rrrrrPr: __classcell__)rTs@rUrrs].f"') '+L9T#Y'L9L9T3Y L9  L9 #s(^ L9L9L9L9L9L9L9$s)$L9L9L9\$eCcN&;$S $L08DI+>04004:C4:D4:E"**eTWY\T\o_dehjmem_nBn "HT#s(^$< ""Dc3h0"rr)$rhrr`rcopyrmultiprocessing.poolrpathlibrtypingrrrr r r r4numpyr|torch.utils.datar ultralytics.data.utilsr rrrultralytics.utilsrrrrrultralytics.utils.patchesrrrrrUr sH +:: $]]PP,d"'d"r