o V hH@sddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z ddl Z ddl ZddlZddlmZddlmZmZmZmZddlmZmZmZmZmZdd lmZGd d d eZdS) N)deepcopy) ThreadPool)Path)Optional)Dataset)FORMATS_HELP_MSGHELP_URL IMG_FORMATScheck_file_speeds) DEFAULT_CFG LOCAL_RANKLOGGER NUM_THREADSTQDM)imreadc seZdZdZdddeddddddd d f fd d Zd dZdeefddZ d*ddZ ddZ ddZ d+ddZ d+ddZddZddZd d!Zd"d#Zd$d%Zd,d&d'Zd(d)ZZS)- BaseDataseta Base dataset class for loading and processing image data. This class provides core functionality for loading images, caching, and preparing data for training and inference in object detection tasks. Attributes: img_path (str): Path to the folder containing images. imgsz (int): Target image size for resizing. augment (bool): Whether to apply data augmentation. single_cls (bool): Whether to treat all objects as a single class. prefix (str): Prefix to print in log messages. fraction (float): Fraction of dataset to utilize. im_files (List[str]): List of image file paths. labels (List[Dict]): List of label data dictionaries. ni (int): Number of images in the dataset. rect (bool): Whether to use rectangular training. batch_size (int): Size of batches. stride (int): Stride used in the model. pad (float): Padding value. buffer (list): Buffer for mosaic images. max_buffer_length (int): Maximum buffer size. ims (list): List of loaded images. im_hw0 (list): List of original image dimensions (h, w). im_hw (list): List of resized image dimensions (h, w). npy_files (List[Path]): List of numpy file paths. cache (str): Cache images to RAM or disk during training. transforms (callable): Image transformation function. Methods: get_img_files: Read image files from the specified path. update_labels: Update labels to include only specified classes. load_image: Load an image from the dataset. cache_images: Cache images to memory or disk. cache_images_to_disk: Save an image as an *.npy file for faster loading. check_cache_disk: Check image caching requirements vs available disk space. check_cache_ram: Check image caching requirements vs available memory. set_rectangle: Set the shape of bounding boxes as rectangles. get_image_and_label: Get and return label information from the dataset. update_labels_info: Custom label format method to be implemented by subclasses. build_transforms: Build transformation pipeline to be implemented by subclasses. get_labels: Get labels method to be implemented by subclasses. iFT ?Ng?csvt||_||_||_| |_||_| |_||j|_ | |_ |j | dt |j |_||_||_| |_| |_|jrI|jdusEJ|g|_|jrZt|j|jddfnd|_dg|jdg|jdg|j|_|_|_dd|j D|_t|tr|n|durd nd|_|jd kr|r|j rt!"d |#n |jd kr|$r|#|j%|d |_&dS) a Initialize BaseDataset with given configuration and options. Args: img_path (str): Path to the folder containing images. imgsz (int, optional): Image size for resizing. cache (bool | str, optional): Cache images to RAM or disk during training. augment (bool, optional): If True, data augmentation is applied. hyp (dict, optional): Hyperparameters to apply data augmentation. prefix (str, optional): Prefix to print in log messages. rect (bool, optional): If True, rectangular training is used. batch_size (int, optional): Size of batches. stride (int, optional): Stride used in the model. pad (float, optional): Padding value. single_cls (bool, optional): If True, single class training is used. classes (list, optional): List of included classes. fraction (float, optional): Fraction of dataset to utilize. ) include_classNircSsg|] }t|dqS)z.npy)r with_suffix).0frI/var/www/vscode/kcb/lib/python3.10/site-packages/ultralytics/data/base.py ~sz(BaseDataset.__init__..Tramzcache='ram' may produce non-deterministic training results. Consider cache='disk' as a deterministic alternative if your disk space allows.disk)hyp)'super__init__img_pathimgszaugment single_clsprefixfraction get_img_filesim_files get_labelslabels update_labelslennirect batch_sizestridepad set_rectanglebufferminmax_buffer_lengthimsim_hw0im_hw npy_files isinstancestrlowercachecheck_cache_ram deterministicr warning cache_imagescheck_cache_diskbuild_transforms transforms)selfr#r$r?r%r r'r0r1r2r3r&classesr( __class__rrr"Cs@ "   ".$ zBaseDataset.__init__c slzg}t|tr |n|gD]Y}t|}|r'|tjt|dddd7}q |r]t|dd#}| }t|j t j |fdd|D7}Wd n1sWwYq t|j|d td d |D}|sJ|jd |dtWnty}zt|jd|dt|d }~ww|jdkr|d tt||j}t||jd|S)aN Read image files from the specified path. Args: img_path (str | List[str]): Path or list of paths to image directories or files. Returns: (List[str]): List of image file paths. Raises: FileNotFoundError: If no images are found or the path doesn't exist. z**z*.*T) recursivezutf-8)encodingcs&g|]}|dr|dn|qS)z./) startswithreplacerxparentrrrs&z-BaseDataset.get_img_files..Nz does not existcss4|]}|ddtvr|dtjVqdS)./N)splitr>r rNosseprOrrr s2z,BaseDataset.get_img_files..zNo images found in z. zError loading data from  )r')r<listris_dirglobr=is_fileopenreadstrip splitlinesrRrWrXFileNotFoundErrorr'sortedr Exceptionrr(roundr.r )rGr#rptr*errQrr)s0  " zBaseDataset.get_img_filesrcst|dd}tt|jD]j}|durj|j|d}|j|d}|j|d|j|d}||kd}|||j|d<|||j|d<r]fdd t|D|j|d<|durj|||j|d<|jrzd |j|dddd f<qdS) z Update labels to include only specified classes. Args: include_class (list, optional): List of classes to include. If None, all classes are included. r[rTNclsbboxessegments keypointscsg|] \}}|r|qSrr)rsiidxrmrrrsz-BaseDataset.update_labels..r) nparrayreshaperanger.r,any enumerater&)rGrinclude_class_arrayirkrlrnjrrqrr-s$ zBaseDataset.update_labelsc Cs(|j||j||j|}}}|dur|rNzt|}Wn0tyM}z t|j d|d|t |j ddt |}WYd}~n d}~wwt |}|dur]t d||jdd\}}|r|jt||} | dkrtt|| |jtt|| |j} } tj|| | ftjd }n||kr|jksntj||j|jftjd }|jr|||f|jdd|j|<|j|<|j|<|j|dt|jkr|jkrnn|jd } |jd krd \|j| <|j| <|j| <|||f|jddfS|j||j||j|fS) a Load an image from dataset index 'i'. Args: i (int): Index of the image to load. rect_mode (bool, optional): Whether to use rectangular resizing. Returns: (np.ndarray): Loaded image as a NumPy array. (Tuple[int, int]): Original image dimensions in (height, width) format. (Tuple[int, int]): Resized image dimensions in (height, width) format. Raises: FileNotFoundError: If the image file is not found. Nz"Removing corrupt *.npy image file z due to: T) missing_okzImage Not Found r[) interpolationrr)NNN) r8r*r;existsrrloadrfr rBr'runlinkrrdshaper$maxr6mathceilcv2resize INTER_LINEARr%r9r:r5appendr.r7popr?) rGry rect_modeimrfnrjh0w0rwhrzrrr load_images@" .0    zBaseDataset.load_imagec Csd\}}|jdkr|jdfn|jdf\}}tt_}||t|j}tt ||jt dkd}|D]:\}} |jdkrE||j | j 7}n| \|j|<|j|<|j|<||j|j7}|jd||dd |d |_q1|Wd d S1s{wYd S) z3Cache images to memory or disk for faster training.ri@rDiskRAMr)totaldisablezCaching images (.1fzGB )N)r?cache_images_to_diskrrrimaprur/rrwr r;statst_sizer8r9r:nbytesr'descclose) rGbgbfcnstoragepoolresultspbarryrPrrrrCs"   " "zBaseDataset.cache_imagescCs8|j|}|stj|t|j|dddSdS)z2Save an image as an *.npy file for faster loading.F) allow_pickleN)r;r~rrsaveas_posixrr*)rGryrrrrrs "z BaseDataset.cache_images_to_diskc Csddl}d\}}t|jd}t|D]/}t|j}t|}|dur#q||j7}t t |j t j sAd|_t|jddSq||j|d|} |t |jdj \} } } | | krd|_t|j| |dd t|d d | |dd | |dd dSdS)a Check if there's enough disk space for caching images. Args: safety_margin (float, optional): Safety margin factor for disk space calculation. Returns: (bool): True if there's enough disk space, False otherwise. rNrz8Skipping caching images to disk, directory not writeableFr[rzGB disk space required, with d% safety margin but only rUz#GB free, not caching images to diskT)shutilr6r/rurandomchoicer*rrrWaccessrrRW_OKr?r rBr' disk_usageint) rG safety_marginrrrn_im_filer disk_requiredrusedfreerrrrDs:      zBaseDataset.check_cache_diskc Csd\}}t|jd}t|D]&}tt|j}|durq|jt|j d|j d}||j |d7}q||j|d|}t } || j krqd|_t|j||ddt|d d | j |dd | j|dd d SdS)z Check if there's enough RAM for caching images. Args: safety_margin (float, optional): Safety margin factor for RAM calculation. Returns: (bool): True if there's enough RAM, False otherwise. rrNrr[r|rz%GB RAM required to cache images with rrrUz GB available, not caching imagesFT)r6r/rurrrr*r$rrrpsutilvirtual_memory availabler?r rBr'rr) rGrrrrrrratio mem_requiredmemrrrr@9s.      zBaseDataset.check_cache_ramc s:ttjjt}|dd}tddjD}|dddf|dddf}| }fdd|D_ fdd|D_||}ddgg|}t |D](}|||k}| | } } | dkrs| dg||<qW| dkrdd| g||<qWtt|jjjtj_|_dS) zBSet the shape of bounding boxes for YOLO detections as rectangles.rTr[cSsg|]}|dqS)r)rrOrrrr\z-BaseDataset.set_rectangle..Nrcg|]}j|qSr)r*rryrGrrr_rcrr)r,rrrrr`r)rrflooraranger/r1astyperrsr,argsortr*rur6rrr$r2r3 batch_shapesbatch) rGbinbsarirectshapesryariminimaxirrrr4Ws&    0 zBaseDataset.set_rectanglecCs|||S)z5Return transformed label information for given index.)rFget_image_and_label)rGindexrrr __getitem__pszBaseDataset.__getitem__cCst|j|}|dd||\|d<|d<|d<|dd|dd|dd|ddf|d<|jrB|j|j||d <||S) z Get and return label information from the dataset. Args: index (int): Index of the image to retrieve. Returns: (dict): Label dictionary with image and metadata. rNimg ori_shape resized_shaperr[ ratio_pad rect_shape)rr,rrr0rrupdate_labels_info)rGrlabelrrrrts  zBaseDataset.get_image_and_labelcCs t|jS)z5Return the length of the labels list for the dataset.)r.r,rrrr__len__s zBaseDataset.__len__cCs|S)zCustom your label format here.r)rGrrrrrszBaseDataset.update_labels_infocCt)a Users can customize augmentations here. Examples: >>> if self.augment: ... # Training transforms ... return Compose([]) >>> else: ... # Val transforms ... return Compose([]) NotImplementedError)rGr rrrrEs zBaseDataset.build_transformscCr)a) Users can customize their own format here. Examples: Ensure output is a dictionary with the following keys: >>> dict( ... im_file=im_file, ... shape=shape, # format: (height, width) ... cls=cls, ... bboxes=bboxes, # xywh ... segments=segments, # xy ... keypoints=keypoints, # xy ... normalized=True, # or False ... bbox_format="xyxy", # or xywh, ltwh ... ) rrrrrr+szBaseDataset.get_labels)T)r)N)__name__ __module__ __qualname____doc__r r"r)rr\r-rrCrrDr@r4rrrrrEr+ __classcell__rrrIrrs:/J& 4  $ r) r^rrWrcopyrmultiprocessing.poolrpathlibrtypingrrnumpyrrrtorch.utils.datarultralytics.data.utilsrrr r ultralytics.utilsr r r rrultralytics.utils.patchesrrrrrrs