|hA ddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z ddl mZmZmZddlZddlZddlmZddlmZddlmcmZddlmZddlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&dd l'm(Z(dd l)m*Z*e(ej:d Z+e(ej:d Z,e(ej:d Z-e(ej:dZ.e(e$dZ/e(e$dZ0e(e$dZ1e(e$dZ2e%r%e(ej:dre!jfdede4fdZ5dZ6d?de7de8fdZ9ejtdZ;ejtdZdZ?d Z@dAd!ZAd"ZBd#ZCd$ZDdBd%ZEdBd&ZFd'ZGdCd(ZHdDd)ZId*ZJdEd+ZKd,ZLd-ZMdFd.ZNdGd/ZOd0ZPGd1d2ZQdHd3ee8e fd4e8d5ee8efd6ee8effd7ZRd8ZSedId9ZTdJd:ZUGd;d<ZVGd=d>ejZXy)KN)contextmanager)deepcopy)datetime)Path)AnyDictUnion) __version__)DEFAULT_CFG_DICTDEFAULT_CFG_KEYSLOGGER NUM_THREADSPYTHON_VERSIONTORCHVISION_VERSIONWINDOWScolorstr) check_version) torch_loadz1.9.0z1.13.0z2.0.0z2.4.0z0.10.0z0.11.0z0.13.0z0.18.0z==2.4.0zKnown issue with torch==2.4.0 on Windows with CPU, recommend upgrading to torch>=2.4.1 to resolve https://github.com/ultralytics/ultralytics/issues/15049 local_rankc#pKtjxrtj}|xrtjdk(}|r1|dvr-|rtj|gntjd|r4|dk(r.|rtj|gntjyyyw)ziEnsure all processes in distributed training wait for the local master (rank 0) to complete a task first.nccl>r) device_idsNr)dist is_availableis_initialized get_backendbarrier)r initializeduse_idss \/var/www/html/test/engine/venv/lib/python3.12/site-packages/ultralytics/utils/torch_utils.pytorch_distributed_zero_firstr"3s##%?$*=*=*?K:d..0F:Gz018  -dlln zQ18  -dlln'{sB4B6c d}|S)zVApply torch.inference_mode() decorator if torch>=1.9.0 else torch.no_grad() decorator.ctrtjr|Strtj|Stj|S)zLApply appropriate torch decorator for inference mode based on torch version.) TORCH_1_9torchis_inference_mode_enabledinference_modeno_grad)fns r!decoratez&smart_inference_mode..decorateCs> 88:IKIE((KBO O5==KBO O)r+s r!smart_inference_moder.@sP Or,enableddevicectr!tjj||Stjjj|S)aI Get the appropriate autocast context manager based on PyTorch version and AMP setting. This function returns a context manager for automatic mixed precision (AMP) training that is compatible with both older and newer versions of PyTorch. It handles the differences in the autocast API between PyTorch versions. Args: enabled (bool): Whether to enable automatic mixed precision. device (str, optional): The device to use for autocast. Returns: (torch.amp.autocast): The appropriate autocast context manager. Notes: - For PyTorch versions 1.13 and newer, it uses `torch.amp.autocast`. - For older versions, it uses `torch.cuda.autocast`. Examples: >>> with autocast(enabled=True): ... # Your mixed precision operations here ... pass )r/) TORCH_1_13r&ampautocastcuda)r/r0s r!r4r4Ms:.yy!!&'!::zz~~&&w//r,cJddlm}d|vry ddl}d}|j}|j |d|vr|dn|d|vr|dn|dd}|j d d j d d j d d |d<|j ddS#t $rYwxYw) z=Return a string with system CPU information, i.e. 'Apple M2'.r)PERSISTENT_CACHEcpu_infoN) brand_raw hardware_rawarch_string_rawunknownz(R)zCPU z@ )ultralytics.utilsr7cpuinfo get_cpu_infogetreplace Exception)r7rAkinfostrings r!rBrBjs3))  >A'')DXXaddlad! !RSTURVXabF+1>>%+D+L+LVUW+X+`+`aegi+j Z (    I 66   sA8B B"!B"ctjj|}|jd|jdz ddS)zGReturn a string with system GPU information, i.e. 'Tesla T4, 15102MiB'., iz.0fMiB)r&r5get_device_propertiesname total_memory)index propertiess r! get_gpu_inforQ|s>11%8Joo b!8!8G!DS I MMr,c ft|tjst|j dr|Sdt dt dtj d}t|j}dD]}|j|d}d|vrd d l m }|jd }|j|jdd }tt|D],} || dk(s |rt|j!d nd|| <.d j#d|D}|dk(} |dv} | s| rdt$j&d<nU|rR|dk(rd}d |vr1d j#|jd D cgc]} | s|  c} }t$j&j)dd} |t$j&d<tj*j-r9tj*j/t|jd k\st1j2|tj*j/d k(rdnd}t5d|dtj*j-dtj*j/d| d| | s| stj*j-r|r|jd nd}t|}|dkDrQ|dkrt5d|d|dzd|d k\r2||zd k7r*t5d |d!|d"||z|zd#||z|z|zd$|d dt|dzz}t7|D]#\} }|| d k(rdn|d%|d&t9| d'z }%d(}nW| rBt:rj-r|d)tAd'z }d*}n|d+tAd'z }d}|d,vrtjBtD|r't1j2|r|n|jGtj|Scc} w)-a% Select the appropriate PyTorch device based on the provided arguments. The function takes a string specifying the device or a torch.device object and returns a torch.device object representing the selected device. The function also validates the number of available devices and raises an exception if the requested device(s) are not available. Args: device (str | torch.device, optional): Device string or torch.device object. Options are 'None', 'cpu', or 'cuda', or '0' or '0,1,2,3'. Auto-selects the first available GPU, or CPU if no GPU is available. batch (int, optional): Batch size being used in your model. newline (bool, optional): If True, adds a newline at the end of the log string. verbose (bool, optional): If True, logs the device information. Returns: (torch.device): Selected device. Raises: ValueError: If the specified device is not available or if the batch size is not a multiple of the number of devices when using multiple GPUs. Examples: >>> select_device("cuda:0") device(type='cuda', index=0) >>> select_device("cpu") device(type='cpu') Notes: Sets the 'CUDA_VISIBLE_DEVICES' environment variable for specifying which GPUs to use. )tpuintelz Ultralytics u 🚀 Python-z torch- )zcuda:none()[]'rUr?z-1r)GPUInfo,g?)countmin_memory_fractionc3&K|] }|s| ywNr-).0ps r! z select_device..s0a!0scpu>mps:0mpsCUDA_VISIBLE_DEVICESr50Nz}See https://pytorch.org/get-started/locally/ for up-to-date torch install instructions if no CUDA devices are seen by torch. zInvalid CUDA 'device=z' requested. Use 'device=cpu' or pass valid CUDA device(s) if available, i.e. 'device=0' or 'device=0,1,2,3' for Multi-GPU. torch.cuda.is_available(): z torch.cuda.device_count(): z% os.environ['CUDA_VISIBLE_DEVICES']:  r<zuAutoBatch with batch<1 not supported for Multi-GPU training, please specify a valid batch size multiple of GPU count z , i.e. batch=.z'batch=z"' must be a multiple of GPU count z . Try 'batch=z ' or 'batch=z/', the nearest batch sizes evenly divisible by zCUDA:z (z) zcuda:0zMPS (rgzCPU (>rerg)$ isinstancer&r0str startswithr rlowerrDultralytics.utils.autodevicer\splitselect_idle_gpur^rangelenpopjoinosenvironrCr5r device_countr rG ValueError enumeraterQ TORCH_2_0backendsrgrBset_num_threadsrrstrip)r0batchnewlineverbosesremover\partsselectedirergxvisibleinstalldevicesnspacedargs r! select_devicers#@&%,,'3v;+A+ABR+S {m=0@HYHYGZZ[\A [   F?,+, v~8 S!9,,5;;t3DZ],^s5z" DAQx43;3x||A/a D0U00 E/C $ $C c-1 )*  V F &=XX&,,s*;AQqqABF**..!7>-3 )* '')ejj.E.E.G3v||\_O`Ka.a KKN::**,14  'x0016 0G0G0I/J/ 0G0G0I/J8 )   suzz668'-&,,s#3 L q5qy OOPcQ^_`cd_d^eefhzeai1n eW$FqcW\`aWadeWeVfg#qj1nq011`ab`ccdfs1vz"g& LDAq !q&Be,E!B|A6GsK KA L u~~11>>@ u\^$C (( u\^$C (( n k* Aahhj1 << gBs 6P.>P.ctjjrtjjt jS)zReturn PyTorch-accurate time.)r&r5r synchronizetimer-r,r! time_syncrs. zz   99;r,c &tj|j|j|j|j |j |j|jdjdj|jj}|jj|jd}tj|jj!tj"|j$|j&z}|jj)tj*||j|jj,|j.Wtj0|jj,d|jj2|jjn |j.}|j.|jj5|j6j!tj"|j&|j$zz }|j.j)tj*||j9ddj9d|z|S)z'Fuse Conv2d() and BatchNorm2d() layers.T) kernel_sizestridepaddingdilationgroupsbiasFrr)dtyper0r<)nnConv2d in_channels out_channelsrrrrrrequires_grad_toweightr0viewr&diagdivsqrteps running_varcopy_mmshaperzerosrmul running_meanreshape)convbn fusedconvw_convw_bnb_convb_bns r!fuse_conv_and_bnrs      ((;;LL]];;    DKK    [[  d// 4F ::biimmEJJrvv/F$GH ID 588D&166y7G7G7M7MNO 99   DKK%%a( 0A0A$++J\J\] YY 77RYY]]2??377 2>>TVTZTZCZ8[\ \D NN$r1(=>FFrJTQR r,c tj|j|j|j|j |j |j|j|jd jdj|jj}|jj|jd}tj |jj#tj$|j&|j(z}|jj+tj,||j|jj.|j0Btj2|jj.d|jjn |j0}|j0|jj5|j6j#tj$|j(|j&zz }|j0j+tj,||j9ddj9d|z|S)z0Fuse ConvTranspose2d() and BatchNorm2d() layers.T)rrroutput_paddingrrrFrr<r0)rConvTranspose2drrrrrrrrrrrr0rr&rrrrrrrrrrrrr)deconvr fuseddconvw_deconvrrrs r!fuse_deconv_and_bnrs       **==NN!00__==    FMM !"}}!!&"5"5r:H ::biimmEJJrvv/F$GH IDEHHT8499*:K:K:Q:QRSRXQ\Q\QdU[[,,Q/ 8L8L MjpjujuF 77RYY]]2??377 2>>TVTZTZCZ8[\ \DOO%((4A)>?GGKdRS r,c,|syt|}t|}tdjd|j D}t |}|rdddddd d d d d dd dd dd }t j|t|jD]A\} \} } | jdd} | jj} t | jr| jD]\} }t j| d| d| d| d |jd |j!dt#t%|j&d |j)d|j+dt#|j,jddd  t j| d| d| d dd ddt#gd dd dd dd Dt/||}t1|ddrdnd}|rd|d d!nd}t1|d"dxst1|d#ij3d"d}t5|j6jd$d%xsd&}t j|d'|d(|d)d*|d)d+|d)d,| ||||fS)-a Print and return detailed model information layer by layer. Args: model (nn.Module): Model to analyze. detailed (bool, optional): Whether to print detailed layer information. verbose (bool, optional): Whether to print model information. imgsz (int | list, optional): Input image size. Returns: n_l (int): Number of layers. n_p (int): Number of parameters. n_g (int): Number of gradients. flops (float): GFLOPs. N collectionsc3\K|]$\}}t|jdk(s||f&yw)rN)ru_modules)rbrms r!rdzmodel_info..Qs-2tda_bcdcmcm_nrs_sAq62ts , ,layerz>5rMz>40typez>20gradientz>10 parametersz>12rmusigmaz module_list.r?z>5grlz>12gz>10.3gztorch.z>15Fr-is_fusedcy)NFr-r-r,r!zmodel_info..bsr,z (fused)rJ.1fz GFLOPs yaml_fileyamlyoloYOLOModelz summaryz: r]z layers, z parameters, z gradients)get_num_paramsget_num_gradients __import__ OrderedDict named_modulesrur rGr|itemsrD __class____name__ _parametersnamed_parameters requires_gradnumelrnlistrmeanstdr get_flopsgetattrrCrstem)modeldetailedrimgszn_pn_glayersn_lhrmnrmtpnrcflopsfusedfsr model_names r! model_infor=s4   C E "C  & 2 22teFYFYF[2t tF f+Crl6#,vcl:c2B.ls5Qqwwy5sumrrs r!rrjs 5%"2"2"45 55r,cBtd|jDS)zEReturn the total number of parameters with gradients in a YOLO model.c3VK|]!}|js|j#ywra)rrrs r!rdz$get_num_gradients..qsHQqwwyHs))rrs r!rros H%"2"2"4H HHr,cx|jjrHddlm}||jg|j j d}|jdn5t|jtt|jdd}t|jjdd|d<|S) a[ Return model info dict with useful model information. Args: trainer (ultralytics.engine.trainer.BaseTrainer): The trainer object containing model and validation data. Returns: (dict): Dictionary containing model parameters, GFLOPs, and inference speeds. Examples: YOLOv8n info for loggers >>> results = { ... "model/parameters": 3151904, ... "model/GFLOPs": 8.746, ... "model/speed_ONNX(ms)": 41.244, ... "model/speed_TensorRT(ms)": 3.211, ... "model/speed_PyTorch(ms)": 18.755, ...} r) ProfileModelsrz model/name)zmodel/parametersz model/GFLOPs inferencezmodel/speed_PyTorch(ms))argsprofileultralytics.utils.benchmarksrlastr0runrvrrroundr validatorspeed)trainerrresultss r!model_info_for_loggersrts(||>w~~FJJLQO L!!/w}} =!)GMM":A> */w/@/@/F/F{/SUV)WG %& Nr,c ddl}|sy t|}t|j }t |t s||g} t|dr-tt|jjdnd}tjd|jd||f|j}|jt!||gd dd z d z}||dz|z |dz|z S#t$rd}YwxYw#t"$r]tjd|jdg||j}|jt!||gd dd z d zcYSwxYw#t"$rYywxYw) a Calculate FLOPs (floating point operations) for a model in billions. Attempts two calculation methods: first with a stride-based tensor for efficiency, then falls back to full image size if needed (e.g., for RTDETR models). Returns 0.0 if thop library is unavailable or calculation fails. Args: model (nn.Module): The model to calculate FLOPs for. imgsz (int | list, optional): Input image size. Returns: (float): The model FLOPs in billions. rNr r<rFinputsreAr=)thop ImportError de_parallelnextrrmrhasattrmaxintrr&emptyrr0rrrE)rrrrcrimrs r!rrsv E" !!# $%&ENE Z9@9QSU\\--/0"5WYFaVV.9AGG9rrc34K|]}|jywrar$rs r!rdz0get_flops_with_torch_profiler..r%r&)r}rrrrmrrrrrr&rrr0profilerrr key_averagesrE)rrrcrr profrs r!get_flops_with_torch_profilerr+s   E U    A eT " @6=eX6N#c%,,**,-r2TVZ[[ [[!QWWQZ8 J ^^ # #t # 4  "I 9T%6%6%899C?a 6)E!H4v= L   @ [[!QWWQZ0%0 B ^^ # #t # 4  "I   9T%6%6%899C? L @sDBD' D?D'D$ D''AG F GF *GGcZ|jD]}t|}|tjur!|tjurd|_d|_B|tjtjtjtjtjhvsd|_ y)z*Initialize model weights to random values.gMbP?gQ?TN) modulesrrr BatchNorm2drmomentum Hardswish LeakyReLUReLUReLU6SiLUinplace)rrts r!initialize_weightsr7sv ]]_ G  >  ".. AEAJ 2<<rww"''J JAIr,c dk(r|S|jdd\}}t|zt|zf}tj||dd}|sfd||fD\}}tj|d||d z d||dz gd S) a Scale and pad an image tensor, optionally maintaining aspect ratio and padding to gs multiple. Args: img (torch.Tensor): Input image tensor. ratio (float, optional): Scaling ratio. same_shape (bool, optional): Whether to maintain the same shape. gs (int, optional): Grid size for padding. Returns: (torch.Tensor): Scaled and padded image tensor. ?r=NbilinearF)sizemode align_cornersc3ZK|]"}tj|zz z$ywra)mathceil)rbrgsratios r!rdzscale_img..s'?1 !e)b.)B.?s(+rr<gS㥛?)value)rrF interpolatepad)imgrB same_shaperArwrs ` ` r! scale_imgrJs | 99QR=DAq QYQY(A --!*E JC ?A?1 55q!ad(Aq1Q4x0 >>r,c|jjD]7\}}t|r||vs|jds||vr+t |||9y)a Copy attributes from object 'b' to object 'a', with options to include/exclude certain attributes. Args: a (Any): Destination object to copy attributes to. b (Any): Source object to copy attributes from. include (tuple, optional): Attributes to include. If empty, all attributes are included. exclude (tuple, optional): Attributes to exclude. _N)__dict__rrurosetattr)abincludeexcluderFvs r! copy_attrrTsR   "1 LQg-!,,s2CqG|  Aq!  r,ctr,tdttjDdz Stjj j ddd}dddddd j|dS) z Return the second-most recent ONNX opset version supported by this version of PyTorch, adjusted for maturity. Returns: (int): The ONNX opset version. c3BK|]}d|vst|ddyw)symbolic_opsetN)r)rbrFs r!rdz#get_latest_opset.. s#R1.2s:W!1A::Ws)rallr)dadbrRrFrSs ` r!intersect_dictsrg&sX XXZ s sTQ17s:Ww:W7W\]\c\cgijkglgrgr\rAqD ss ss AAAAc~t|tjjtjjfS)z Return True if model is of type DP or DDP. Args: model (nn.Module): Model to check. Returns: (bool): True if model is DataParallel or DistributedDataParallel. )rmrparallel DataParallelDistributedDataParallelrs r! is_parallelrl5s* ebkk66 8[8[\ ]]r,c4t|r |jS|S)z De-parallelize a model: return single-GPU model if model is of type DP or DDP. Args: model (nn.Module): Model to de-parallelize. Returns: (nn.Module): De-parallelized model. )rlmodulers r!rrBs'u-5<<858r,cfdS)aR Return a lambda function for sinusoidal ramp from y1 to y2 https://arxiv.org/pdf/1812.01187.pdf. Args: y1 (float, optional): Initial value. y2 (float, optional): Final value. steps (int, optional): Number of steps. Returns: (function): Lambda function for computing the sinusoidal ramp. ctdtj|tjzz z dz dz zzS)Nr<r=r)rr?cospi)rstepsy1y2s r!rzone_cycle..[s=S!dhhq477{U':;;q@!DRPSUUr,r-)rtrurss```r! one_cyclervOs  VUr,c.tj|tjj|tj|tj j |tj j ||rtrftjdddtjj_ dtjd<t|tjd<ytj dyt#y)z Initialize random number generator (RNG) seeds https://pytorch.org/docs/stable/notes/randomness.html. Args: seed (int, optional): Random seed. deterministic (bool, optional): Whether to set deterministic algorithms. T) warn_onlyz:4096:8CUBLAS_WORKSPACE_CONFIGPYTHONHASHSEEDz3Upgrade to torch>=2.0.0 for deterministic training.N)randomseednpr& manual_seedr5manual_seed_allr}use_deterministic_algorithmsr~cudnn deterministicrxryrnr warningunset_deterministic)r|rs r! init_seedsr^s KKIINN4 d JJ4  JJt$   . .tt D15ENN .4=BJJ0 1+.t9BJJ' ( NNP Qr,ctjddtjj_t j jddt j jddy)z@Unset all the configurations applied for deterministic training.FryNrz)r&rr~rrrxryrvr-r,r!rrxsH &&u-).ENN&JJNN,d3JJNN#T*r,c&eZdZdZddZdZddZy)ModelEMAa Updated Exponential Moving Average (EMA) implementation. Keeps a moving average of everything in the model state_dict (parameters and buffers). For EMA details see References. To disable EMA set the `enabled` attribute to `False`. Attributes: ema (nn.Module): Copy of the model in evaluation mode. updates (int): Number of EMA updates. decay (function): Decay function that determines the EMA weight. enabled (bool): Whether EMA is enabled. References: - https://github.com/rwightman/pytorch-image-models - https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage ctt|j|_||_fd|_|jj D]}|jdd|_y)a@ Initialize EMA for 'model' with given arguments. Args: model (nn.Module): Model to create EMA for. decay (float, optional): Maximum EMA decay rate. tau (int, optional): EMA decay time constant. updates (int, optional): Initial number of updates. cBdtj| z z zS)Nr<)r?exp)rdecaytaus r!rz#ModelEMA.__init__..s uDHHaR#X,>(>?r,FTN) rrevalemaupdatesrrrr/)selfrrrrrcs `` r!__init__zModelEMA.__init__s^K./446 ? $$& $A  U # $ r,c|jr|xjdz c_|j|j}t|j }|j j j D]=\}}|jjs||z}|d|z ||jzz }?yy)zp Update EMA parameters. Args: model (nn.Module): Model to update EMA from. r<N) r/rrr state_dictrrris_floating_pointdetach)rrrmsdrFrSs r!updatezModelEMA.updates << LLA L 4<<(Ae$//1C++-335 3177,,FA!a%3q6==?22A 3 r,cN|jrt|j|||yy)a Update attributes and save stripped model with optimizer removed. Args: model (nn.Module): Model to update attributes from. include (tuple, optional): Attributes to include. exclude (tuple, optional): Attributes to exclude. N)r/rTr)rrrQrRs r! update_attrzModelEMA.update_attrs" << dhhw 8 r,N)gH.?ir)r-) process_groupreducer)r __module__ __qualname____doc__rrrr-r,r!rrs&"3$ 9r,rfrrreturnc  t|tjd}t|tsJdd|vsJd tjjtd d d }|jd r|d |d<t|dd r t |dj|d_t|ddr d|d_|dj!|dj#D] }d|_ it&|jdi}dD]}d||< d|d<|j)D cic]\}} |t*vs|| c} }|d<i|||xsi} tj,| |xs|t.j0j3|xs|dz } t j4d|d|rd|dndd| dd| S#t $r'}t jd|d|icYd}~Sd}~wwxYwcc} }w)a Strip optimizer from 'f' to finalize training, optionally save as 's'. Args: f (str | Path): File path to model to strip the optimizer from. s (str, optional): File path to save the model with stripped optimizer to. If not provided, 'f' will be overwritten. updates (dict, optional): A dictionary of updates to overlay onto the checkpoint before saving. Returns: (dict): The combined checkpoint dictionary. Examples: >>> from pathlib import Path >>> from ultralytics.utils.torch_utils import strip_optimizer >>> for f in Path("path/to/model/checkpoints").rglob("*.pt"): >>> strip_optimizer(f) re) map_locationz%checkpoint is not a Python dictionaryrz'model' missing from checkpointz Skipping z!, not a valid Ultralytics model: Nz2AGPL-3.0 License (https://ultralytics.com/license)zhttps://docs.ultralytics.com)dater`licensedocsrr criterionF train_args) optimizer best_fitnessrrrepochg.AzOptimizer stripped from r]z saved as r?rUrMB)rr&r0rmdictrEr rrnow isoformatr rCrrrhalfrrr rr saverxpathgetsizerG) rrrremetadatarcrrFrScombinedmbs r!strip_optimizerrs& qu||E': ;!T"K$KK"!|>>>|  ((*G. H uuU|uX' qz6"qz/' qz;'#' gJOO wZ " " $   ; :!%% b"9 :D :!AgJ(, N1=M8Mq!tNAlO4(3a3GMr3H JJxa a 3 &B KK*1#QAA3a/@2.NaPRSVxWYZ[ OG 1#%FqcJK 6Os*AGHH HG?9H?Hc|djD]k}|jD]V\}}|dk7s t|tjs'|j tj usD|j||<Xm|S)a  Convert the state_dict of a given optimizer to FP16, focusing on the 'state' key for tensor conversions. Args: state_dict (dict): Optimizer state dictionary. Returns: (dict): Converted optimizer state dictionary with FP16 tensors. statestep)valuesrrmr&Tensorrfloat32r)rrrFrSs r!$convert_optimizer_state_dict_to_fp16rstG$++-$KKM $DAqF{z!U\\:qww%--?W668a $$ r,c#BKtd}tjjrFtjj  |tjj ||d<y|y#tjj ||d<wxYww)aN Monitor and manage CUDA memory usage. This function checks if CUDA is available and, if so, empties the CUDA cache to free up unused memory. It then yields a dictionary containing memory usage information, which can be updated by the caller. Finally, it updates the dictionary with the amount of memory reserved by CUDA on the specified device. Args: device (torch.device, optional): The CUDA device to query memory usage for. Yields: (dict): A dictionary with a key 'memory' initialized to 0, which will be updated with the reserved memory. r)memoryrN)rr&r5r empty_cachememory_reserved)r0 cuda_infos r!cuda_memory_usagersvAI zz    EO"'**"<">> from ultralytics.utils.torch_utils import profile_ops >>> input = torch.randn(16, 3, 640, 640) >>> m1 = lambda x: x * torch.sigmoid(x) >>> m2 = nn.SiLU() >>> profile_ops(input, [m1, m2], n=100) # profile over 100 iterations rNParamsz>12sGFLOPsz GPU_mem (GB)z>14sz forward (ms)z backward (ms)inputz>24soutputTrr)rrrFrrr=r<c3<K|]}|jywra)r)rbyis r!rdzprofile_ops..ds 6b 6rnanric3hK|])}jd|z jd|z z+yw)rN)r)rbrrs r!rdzprofile_ops..qs/'iRSqQWWR[1_(M'is/2)r0rc3~K|]5}t|tjrt|jnd7yw)rN)rmr&rtuplerrs r!rdzprofile_ops..vs,i]^Au||1LuQWW~RXXis;=c3<K|]}|jywrarrs r!rdzprofile_ops..ws:a :r12z12.4gz>14.3fz14.4g)(rrrmr&r0rr rGgccollectr5rrrrrrrfloat16rrrrErtrrrbackwardfloatrandnrrrtolistrrModulerrnappend)ropsrr0 max_num_objrrrtftbr6rmemrLrys_ins_outrcrrs @r! profile_opsr-s(*G fell +v& KK D/(4(=nT=RSbcgRh 4.$ )JJL JJ -UE7/) DDL"3-C5, )A '4 0V aA#Av.:a3NSTSZSZ^c^k^kSkqrA1iAB ]a Xa[!e LQORUUXYYgh# )q9A*62 0i({!aD({!0:DQ:MS 6A 66STYY[ddf#,;AaD 09X.44C1Q4!A$;$.22B1Q4!A$;$.22B".v6)!KK ! + #C'iWXW_W_WfWfWh'i$i j'-&+mm y2S88/90jcdfgbhi e>HBII>VC:1<<>::\] qfU5M#fbZ5zRUVZR[\`Qabefkblmqarst5#r2tUCD   &&(Y, )/)` Ny *   )0#(&P94Q9P>>Q4Q5ceZdZdZddZdZy) EarlyStoppinga Early stopping class that stops training when a specified number of epochs have passed without improvement. Attributes: best_fitness (float): Best fitness value observed. best_epoch (int): Epoch where best fitness was observed. patience (int): Number of epochs to wait after fitness stops improving before stopping. possible_stop (bool): Flag indicating if stopping may occur next epoch. cVd|_d|_|xs td|_d|_y)z Initialize early stopping object. Args: patience (int, optional): Number of epochs to wait after fitness stops improving before stopping. rrinfFN)r best_epochrpatience possible_stop)rrs r!rzEarlyStopping.__init__s+  0E%L "r,c f|y||jkDs|jdk(r||_||_||jz }||jdz k\|_||jk\}|rJt d}t j |d|jd|jd|jd|S) a Check whether to stop training. Args: epoch (int): Current epoch of training fitness (float): Fitness value of current epoch Returns: (bool): True if training should stop, False otherwise Frr<zEarlyStopping: z:Training stopped early as no improvement observed in last z( epochs. Best results observed at epoch z@, best model saved as best.pt. To update EarlyStopping(patience=z^) pass a new patience value, i.e. `patience=300` or use `patience=0` to disable EarlyStopping.)rrrrrr rG)rrfitnessdeltastopprefixs r!__call__zEarlyStopping.__call__s ? T&& &$*;*;q*@#DO 'D '"t}}q'89 % /0F KK(TUYUbUbTcd226//1BC448MM?CTU   r,N)2)rrrrrrr-r,r!rrs #r,rc(eZdZdZfdZdZxZS)FXModela} A custom model class for torch.fx compatibility. This class extends `torch.nn.Module` and is designed to ensure compatibility with torch.fx for tracing and graph manipulation. It copies attributes from an existing model and explicitly sets the model attribute to ensure proper copying. Attributes: model (nn.Module): The original model's layers. c\t|t|||j|_y)z Initialize the FXModel. Args: model (nn.Module): The original model to wrap for torch.fx compatibility. N)superrrTr)rrrs r!rzFXModel.__init__s% $[[ r,c g}|jD]w}|jdk7rMt|jtr||jn#|jDcgc]}|dk(r|n||c}}||}|j |y|Scc}w)aa Forward pass through the model. This method performs the forward pass through the model, handling the dependencies between layers and saving intermediate outputs. Args: x (torch.Tensor): The input tensor to the model. Returns: (torch.Tensor): The output tensor from the model. r)rrrmrr)rrrrjs r!forwardzFXModel.forwards  Assby(c2AaccFYZY\Y\8]TUa2g1Q49O8]!A HHQK   9^sB )rrrrrr __classcell__)rs@r!rrs  !r,r)r5)r?rFT)FT)r)r9Fr)r-r-)r-)rr9d)rF)zbest.ptr?Nra) Nr)Y functoolsrr?rxr{r contextlibrcopyrrpathlibrtypingrrr numpyr}r&torch.distributed distributedrtorch.nnrtorch.nn.functional functionalrD ultralyticsr r@r r r rrrrrultralytics.utils.checksrultralytics.utils.patchesrr%r2r} TORCH_2_4TORCHVISION_0_10TORCHVISION_0_11TORCHVISION_0_13TORCHVISION_0_18rrr"r.boolrnr4 lru_cacherBrQrrrrrrrrrr+r7rJrTrargrlrrvrrrrrrrrrrr-r,r!rsf %## #   30 %++W 5 5,,h 7 %++W 5 %++W 5  !4h? !4h? !4h? !4h? }U.. :FNN B  MS M M 0d0C0: 77" NN odD>* Z6 I D'TD ?." W t ^ 9 V4+A9A9H:uS$Y'::4PSUXPX>:eijmorjres:z$2Sl33l,bii,r,