
    hi                        d Z ddlZddlZddlZddlZddlZddlZddlmZmZ ddl	m	Z	m
Z
 ddlmZ ddlZddlZddlmZ ddlmZmZ ddlmZ dd	lmZmZ dd
lmZmZ ddlmZmZ ddlm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6 ddl7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZB  G d d      ZCy)zz
Train a model on a dataset.

Usage:
    $ yolo mode=train model=yolo11n.pt data=coco8.yaml imgsz=640 epochs=100 batch=16
    N)copydeepcopy)datetime	timedelta)Path)distributed)nnoptim)__version__)get_cfgget_save_dir)check_cls_datasetcheck_det_dataset)attempt_load_one_weightattempt_load_weights)
DEFAULT_CFG
LOCAL_RANKLOGGERRANKTQDMYAML	callbacks	clean_urlcolorstremojis)check_train_batch_size)	check_amp
check_filecheck_imgszcheck_model_file_from_stem
print_args)ddp_cleanupgenerate_ddp_command)get_latest_run)	TORCH_2_4EarlyStoppingModelEMAautocast$convert_optimizer_state_dict_to_fp16
init_seeds	one_cycleselect_devicestrip_optimizertorch_distributed_zero_firstunset_deterministicc                   (   e Zd ZdZeddfdZdefdZdefdZdefdZ	d Z
d	 Zd
 Zd Zd+dZd,dZd-dZd.defdZd Zd Zd Zd Zd Zd Zd Zd Zd/dZd Zd0dZd1dZd2dZd Z d Z!d  Z"d! Z#d" Z$d# Z%d$ Z&d.d%Z'd& Z(d' Z)d( Z*d) Z+d3d*Z,y)4BaseTrainera  
    A base class for creating trainers.

    This class provides the foundation for training YOLO models, handling the training loop, validation, checkpointing,
    and various training utilities. It supports both single-GPU and multi-GPU distributed training.

    Attributes:
        args (SimpleNamespace): Configuration for the trainer.
        validator (BaseValidator): Validator instance.
        model (nn.Module): Model instance.
        callbacks (defaultdict): Dictionary of callbacks.
        save_dir (Path): Directory to save results.
        wdir (Path): Directory to save weights.
        last (Path): Path to the last checkpoint.
        best (Path): Path to the best checkpoint.
        save_period (int): Save checkpoint every x epochs (disabled if < 1).
        batch_size (int): Batch size for training.
        epochs (int): Number of epochs to train for.
        start_epoch (int): Starting epoch for training.
        device (torch.device): Device to use for training.
        amp (bool): Flag to enable AMP (Automatic Mixed Precision).
        scaler (amp.GradScaler): Gradient scaler for AMP.
        data (str): Path to data.
        ema (nn.Module): EMA (Exponential Moving Average) of the model.
        resume (bool): Resume training from a checkpoint.
        lf (nn.Module): Loss function.
        scheduler (torch.optim.lr_scheduler._LRScheduler): Learning rate scheduler.
        best_fitness (float): The best fitness value achieved.
        fitness (float): Current fitness value.
        loss (float): Current loss value.
        tloss (float): Total loss value.
        loss_names (list): List of loss names.
        csv (Path): Path to results CSV file.
        metrics (dict): Dictionary of metrics.
        plots (dict): Dictionary of plots.

    Methods:
        train: Execute the training process.
        validate: Run validation on the test set.
        save_model: Save model training checkpoints.
        get_dataset: Get train and validation datasets.
        setup_model: Load, create, or download model.
        build_optimizer: Construct an optimizer for the model.

    Examples:
        Initialize a trainer and start training
        >>> trainer = BaseTrainer(cfg="config.yaml")
        >>> trainer.train()
    Nc                 :   t        ||      | _        | j                  |       t        | j                  j                  | j                  j
                        | _        dt        | j                        v rt        j                  d      nt        | j                        | j                  _        d| _	        d| _
        i | _        t        | j                  j                  dz   t        z   | j                  j                         t!        | j                        | _        | j"                  j$                  | j                  _        | j"                  dz  | _        t        dv rw| j&                  j)                  dd	       t        | j"                        | j                  _        t+        j,                  | j"                  d
z  t/        | j                               | j&                  dz  | j&                  dz  c| _        | _        | j                  j4                  | _        | j                  j
                  | _        | j                  j8                  xs d| _        d| _        t        dk(  rt=        t/        | j                               | j                  j>                  dv rd| j                  _         tC        | j                  jD                        | _"        tG        tH              5  | jK                         | _&        ddd       d| _'        d| _(        d| _)        d| _*        d| _+        d| _,        d| _-        dg| _.        | j"                  dz  | _/        g d| _0        d| _1        |xs te        jf                         | _2        t        dv rte        jh                  |        yy# 1 sw Y   xY w)a  
        Initialize the BaseTrainer class.

        Args:
            cfg (str, optional): Path to a configuration file.
            overrides (dict, optional): Configuration overrides.
            _callbacks (list, optional): List of callback functions.
        cudaCUDA_VISIBLE_DEVICESN   )deterministicweights   r   T)parentsexist_okz	args.yamlzlast.ptzbest.ptd   r   r9   >   cpumpsLosszresults.csv)r   r5      )5r   argscheck_resumer,   devicebatchstrosgetenv	validatormetricsplotsr*   seedr   r6   r   save_dirnamewdirmkdirr   savevarslastbestsave_period
batch_sizeepochsstart_epochr!   typeworkersr    modelr.   r   get_datasetdataemalf	schedulerbest_fitnessfitnesslosstloss
loss_namescsvplot_idxhub_sessionr   get_default_callbacksadd_integration_callbacks)selfcfg	overrides
_callbackss       X/var/www/html/eduruby.in/venv/lib/python3.12/site-packages/ultralytics/engine/trainer.py__init__zBaseTrainer.__init__n   s    C+	)$#DII$4$4diiooF@F#dkkJZ@Z299%;<`cdhdodo`p		
499>>A%,DII<S<ST %TYY/++		MMI-	7?IIOOD4O8!$T]]!3DIIIIdmmk14		?C#yy94dii)6K	499900))//ii&&-#2:tDII' ;;~- !DII 0		@
)*5 	+((*DI	+   !	
!(===0!   $Hy'F'F'H7?//5 /	+ 	+s   NNeventc                 @    | j                   |   j                  |       y)z7Append the given callback to the event's callback list.N)r   appendrj   rp   callbacks      rn   add_callbackzBaseTrainer.add_callback   s    u$$X.    c                 $    |g| j                   |<   y)zPOverride the existing callbacks with the given callback for the specified event.N)r   rs   s      rn   set_callbackzBaseTrainer.set_callback   s    !)
urv   c                 V    | j                   j                  |g       D ]
  } ||         y)z>Run all existing callbacks associated with a particular event.N)r   getrs   s      rn   run_callbackszBaseTrainer.run_callbacks   s)    **5"5 	HTN	rv   c                 x   t        | j                  j                  t              rNt	        | j                  j                        r/t	        | j                  j                  j                  d            }nt        | j                  j                  t        t        f      r t	        | j                  j                        }n>| j                  j                  dv rd}n#t        j                  j                         rd}nd}|dkD  rdt        j                  vr| j                  j                  r&t        j                  d       d| j                  _        | j                  j                   dk  r&t        j                  d	       d
| j                  _        t#        ||       \  }}	 t        j$                  t'        d       ddj)                  |              t+        j,                  |d       	 t1        | t        |             y| j3                  |       y# t.        $ r}|d}~ww xY w# t1        | t        |             w xY w)zIAllow device='', device=None on Multi-GPU systems to default to device=0.,>   r=   r>   r   r5   r   zI'rect=True' is incompatible with Multi-GPU training, setting 'rect=False'F      ?z['batch<1' for AutoBatch is incompatible with Multi-GPU training, setting default 'batch=16'   zDDP:z debug command  T)checkN)
isinstancerA   rC   rE   lensplittuplelisttorchr3   is_availablerF   environrectr   warningrD   r#   infor   join
subprocessrun	Exceptionr"   	_do_train)rj   
world_sizecmdfilees        rn   trainzBaseTrainer.train   s   dii&&,TYY5E5E1FTYY--33C89J		((5$-8TYY--.JYY/JZZ$$&JJ >l"**<yy~~jk!&		yy$q #%		 -Z>IC-x/0OPs$/ D#d), NN:&   D#d),s%   A	H 	HHHH" "H9c                      j                   j                  r1t        d j                   j                   j                         _        n
 fd _        t        j                  j                   j                   j
                         _
        y)z,Initialize training learning rate scheduler.r5   c                     t        d| j                  z  z
  d      dj                  j                  z
  z  j                  j                  z   S )Nr5   r   r~   )maxrV   rA   lrf)xrj   s    rn   <lambda>z.BaseTrainer._setup_scheduler.<locals>.<lambda>   s?    ADKK$7 ;sTYY]]?R SVZV_V_VcVc c rv   )	lr_lambdaN)rA   cos_lrr+   r   rV   r^   r
   lr_schedulerLambdaLR	optimizerr_   rj   s   `rn   _setup_schedulerzBaseTrainer._setup_scheduler   sX    99499==$++>DGcDG++44T^^tww4Wrv   c                 (   t         j                  j                  t               t        j                  dt              | _        dt
        j                  d<   t        j                  t        j                         rdndt        d      t        |       y	)
zGInitialize and set the DistributedDataParallel parameters for training.r3   1TORCH_NCCL_BLOCKING_WAITncclglooi0*  )seconds)backendtimeoutrankr   N)r   r3   
set_devicer   rC   rF   r   distinit_process_groupis_nccl_availabler   )rj   r   s     rn   
_setup_ddpzBaseTrainer._setup_ddp   sa    

d#ll64014

-."446FFe,!		
rv   c           	         | j                  d       | j                         }| j                  j                  | j                        | _        | j                          t        | j                  j                  t              r| j                  j                  nDt        | j                  j                  t              rt        | j                  j                        ng }dg}|D cg c]  }d| d
 c}|z   }|| _        | j                  j                         D ]~  \  }t        fd|D              r!t        j                   d d       d|_        ;|j"                  rH|j$                  j&                  s_t        j(                  d	 d
       d|_         t+        j,                  | j                  j.                        j                  | j                        | _        | j.                  rjt0        dv rbt2        j4                  j7                         }t+        j,                  t9        | j                        | j                        | _        |t2        _        t0        dkD  r4|dkD  r/t;        j<                  | j.                  j                         d       t?        | j.                        | _        t@        r+t*        j.                  jC                  d| j.                        n3t*        jD                  j.                  jC                  | j.                        | _#        |dkD  r6tH        jJ                  jM                  | j                  t0        gd      | _        tO        t        tQ        | j                  d      r$| j                  jR                  jO                         nd      d      }	tU        | j                  jV                  |	|	d      | j                  _+        |	| _)        | jX                  dk  r/t0        dk(  r&| j[                         x| j                  _.        | _,        | jX                  tO        |d      z  }
| j_                  | j`                  d   |
tb        d      | _2        t0        dv r| j_                  | j`                  jg                  d      xs | j`                  jg                  d      | j                  jh                  dk(  r|
n|
dz  dd      | _5        | jm                         | _7        | jn                  jp                  jr                  | ju                  d      z   }tw        ty        |dgt{        |      z              | _8        t}        | j                        | _?        | j                  j                  r| j                          tO        t        | j                  j                  | jX                  z        d      | _D        | j                  j                  | jX                  z  | j                  z  | j                  j                  z  }t        j                  t{        | jd                  j                        tO        | jX                  | j                  j                        z        | j                  z  }| j                  | j                  | j                  j                  | j                  j                  | j                  j                  ||      | _K        | j                          t        | j                  j                         dc| _Q        | _R        | j                  |       | j                  dz
  | j                  _V        | j                  d!       y"c c}w )#z8Build dataloaders and optimizer on correct rank process.on_pretrain_routine_startz.dflzmodel..c              3   &   K   | ]  }|v  
 y wN ).0r   ks     rn   	<genexpr>z+BaseTrainer._setup_train.<locals>.<genexpr>  s     6a166s   zFreezing layer ''Fz/setting 'requires_grad=True' for frozen layer 'zE'. See ultralytics.engine.trainer for customization of frozen layers.Tr8   )rC   r9   r5   r   )srcr3   )enabled)
device_idsfind_unused_parametersstride    )r   floormax_dimr   )rU   r   modevaltestobbr@   )prefix)rZ   rM   lrmomentumdecay
iterations)patienceon_pretrain_routine_endN)Wr{   setup_modelrZ   torC   set_model_attributesr   rA   freezer   intrangefreeze_layer_namesnamed_parametersanyr   r   requires_graddtypeis_floating_pointr   r   tensorampr   r   default_callbacksr   r   r   	broadcastboolr%   
GradScalerr3   scalerr	   parallelDistributedDataParallelr   hasattrr   r   imgszrU   
auto_batchrD   get_dataloaderr\   r   train_loaderrz   tasktest_loaderget_validatorrH   rI   keyslabel_loss_itemsdictzipr   r'   r]   rJ   plot_training_labelsroundnbs
accumulateweight_decaymathceildatasetrV   build_optimizerr   lr0r   r   r&   r   stopperstopresume_trainingrW   r_   
last_epoch)rj   r   ckptfreeze_listalways_freeze_namesr   r   vcallbacks_backupgsrU   metric_keysr   r   r   s                 @rn   _setup_trainzBaseTrainer._setup_train   sn    	67!ZZ]]4;;/
!!#
 $))**D1 II $))**C0 tyy''( 	  &h5@AqcmADWW"4JJ//1 
	'DAq6#566.qc34"'__)B)BEaS IY Y #'
	' <<		.11$++>88(::??A||Idjj$9$++NDH*:I'"9aNN488<<>q1>>GEII   :UZZ^^MfMfosowowMfMx 	 ><<TZZUYTZsw<xDJ 

H0MTZZ&&**,SUVXZ[%diioobTUV		 ??Q42:040AADIIOdo __J(::
 //IIg:JW 0 
 7?#22		e$=		f(=)-5)@:jSTn	  3  D "//1DN..00558M8MUZ8M8[[KK!s;7G1G HIDL

+DHyy))+ eDIIMMDOO$CDaHyy--?$//QTXT]T]TaTaaYYs4#4#4#<#<=DOOUYU^U^UbUb@ccdgkgrgrr
--**$$yy}}YY''! . 
 	"/9K9K"LediT"$($4$4q$8!45W Bs   #]c                    |dkD  r| j                  |       | j                  |       t        | j                        }| j                  j
                  dkD  r,t        t        | j                  j
                  |z        d      nd}d}d| _        t        j                         | _
        t        j                         | _        | j                  d       t        j                  d| j                  j                   d| j                  j                   d	| j                  j                   |xs dz   d
t#        d| j$                         d	| j                  j                  r| j                  j                   dn| j&                   dz          | j                  j(                  rJ| j&                  | j                  j(                  z
  |z  }| j*                  j-                  ||dz   |dz   g       | j.                  }| j0                  j3                          	 || _        | j                  d       t7        j8                         5  t7        j:                  d       | j<                  j?                          ddd       | jA                          tB        dk7  r%| j                  jD                  jG                  |       tI        | j                        }|| j&                  | j                  j(                  z
  k(  r*| jK                          | j                  jM                          tB        dv rCt        j                  | jO                                tQ        tI        | j                        |      }d| _)        |D ]0  \  }}	| j                  d       |||z  z   }
|
|k  r d|g}t        dtU        tW        jX                  |
|d| j                  jZ                  | j\                  z  g      j                                     | _/        tI        | j0                  j`                        D ]  \  }}tW        jX                  |
||dk(  r| j                  jb                  nd|d   | je                  |      z  g      |d<   d|v sWtW        jX                  |
|| j                  jf                  | j                  jh                  g      |d<    tk        | jl                        5  | jo                  |	      }	| jq                  |	      \  }| _9        |ju                         | _;        tB        dk7  r| xjv                  |z  c_;        | jR                  "| jR                  |z  | jr                  z   |dz   z  n| jr                  | _)        ddd       | jx                  j{                  | jv                        j}                          |
|z
  | j^                  k\  r| j                          |
}| j                  j                  rt        j                         | j                  z
  | j                  j                  dz  kD  | _@        tB        dk7  r8tB        dk(  r| j                  ndg}t        j                  |d       |d   | _@        | j                  r n7tB        dv rt        | jR                  j                        r| jR                  j                  d   nd}|j                  ddd|z   z  z   |dz    d| j&                   | j                         ddg|dkD  r| jR                  nt        j                  | jR                  d      |	d   j                  d   |	d    j                  d   z         | j                  d!       | j                  j                  r |
| j*                  v r| j                  |	|
       | j                  d"       3 tI        | j0                  j`                        D ci c]  \  }}d#| |d    c}}| _J        | j                  d$       tB        dv r|dz   | j&                  k\  }| j                  j                  | jp                  g d%&       | j                  j                  s$|s"| j                  j                  s| j                  r/| j                  d'(       | j                         \  | _R        | _S        | j                  i | j                  | jR                        | j                  | j                  )       | xj                  | j                  |dz   | j                        xs |z  c_@        | j                  j                  rN| xj                  t        j                         | j                  z
  | j                  j                  dz  kD  z  c_@        | j                  j                  s|r!| j                          | j                  d*       t        j                         }|| j                  z
  | _        || _
        | j                  j                  r|| j                  z
  || j.                  z
  dz   z  }t        j                  | j                  j                  dz  |z        x| _        | j                  _        | j                          | j4                  | j<                  _[        | xj                  || j&                  k\  z  c_@        | j                  d+       | j                  d'       tB        dk7  r8tB        dk(  r| j                  ndg}t        j                  |d       |d   | _@        | j                  rn|dz  }	tB        dv rt        j                         | j                  z
  }t        j                  d,|| j.                  z
  dz    d-|dz  d.d/       | j                          | j                  j                  r| j                          | j                  d0       | j                          t                | j                  d1       y# 1 sw Y   	xY w# 1 sw Y   xY wc c}}w )2z.Train the model with the specified world size.r5   r   r<   r9   Non_train_startzImage sizes z train, z val
Using z' dataloader workers
Logging results to boldz
Starting training for z	 hours...z
 epochs...r@   on_train_epoch_startignorer8   )totalon_train_batch_start        
initial_lrr   r   i  z%11s%11sz%11.4g/z.3gGclsimgon_batch_endon_train_batch_endzlr/pgon_train_epoch_end)yamlncrA   namesr   class_weights)includeg      ?)	threshold)rI   on_model_saveon_fit_epoch_end
z epochs completed in z.3fz hours.on_train_endteardown)_r   r  r   r   rA   warmup_epochsr   r   
epoch_timetimeepoch_time_starttrain_time_startr{   r   r   r   num_workersr   rL   rV   close_mosaicrf   extendrW   r   	zero_gradepochwarningscatch_warningssimplefilterr_   step_model_trainr   sampler	set_epoch	enumerate_close_dataloader_mosaicresetprogress_stringr   rc   r   npinterpr   rU   r   param_groupswarmup_bias_lrr^   warmup_momentumr   r(   r   preprocess_batchrZ   
loss_itemssumrb   r   scalebackwardoptimizer_stepr   r   broadcast_object_listshapeset_description_get_memoryr   	unsqueezerJ   plot_training_samplesr   r]   update_attrr   r   possible_stop_clear_memoryvalidaterI   ra   save_metricsr   rP   
save_modelr   r   r   r   
final_evalplot_metricsr/   )rj   r   nbnwlast_opt_stepbase_idxr*  pbarirD   nixijr   rb   broadcast_listloss_lengthirfinal_epochtmean_epoch_timer   s                         rn   r   zBaseTrainer._do_trainX  s	   >OOJ'*%""#>Bii>U>UXY>YStyy..34c:_a $		 $		+,499??+8DIIOO3D E&&22joAFG H""*64=="A!B C%& JNDIINN+;9)E`d`k`k_llv]wy	
 99!!dii&<&<<BHMM  (HqL(Q,!GH    "DJ56((* &%%h/##%& rz!!))33E:T../Dtyy'='==>--/!!'')wD0023Id&7&78CDJ  =95""#9:e^8RB&)!S2rAtyy}}W[WfWfGfCg1h1n1n1p-q&rDO )$..*E*E F o1"$))aTYY%=%=SRST`RadhdkdklqdrRr$s#$ &?,.IIb"tyy?X?XZ^ZcZcZlZl>m,nAjMo dhh'  11%8E,0JJu,=)D$/ $
DIrz		Z/	HL

H^a$//9a!eDdhdsds J !!$)),557 %8'')$&M yy~~%)YY[43H3H%HTYY^^^bMb$c	2:;?19dii$-ON 66~qI(6q(9DI99! 7?9<TZZ=M=M9N$**"2"21"5TUK((#h!k/&BB$qyk4;;-8#//1#6a8 -8!OdjjQUQ[Q[]^A_ "%L..q1	
 "%L..r2	 &&~6yy2+>225"=""#78{=9~ ;DDNND_D_:`aQrd|QtW,aDG34w#ai4;;6$$TZZ9s$t 99==K4<<3M3MQUQZQZ&&&515.DL$,!!*jT-B-B4::-N*jRVR^R^*jbfbibi*j!k		T\\%!)T\\BQkQ	99>>II$))+0E0E"E$))..[_J_!``I 99>>[OO%&&7 		A$"7"77DO$%D!yy~~#$t'<'<#<IYIYAY\]A]"^15499>>D;PSb;b1ccdii.%%',0JJ)		Udkk11	12s# rz/3qy$))d!C**>1=*1-	yyQJEw z 7?iikD$9$99GKK"UT%5%559::OPWZ^P^_bOccjklOOyy!!#~.:&I& &B ^ bs   ;0o.Bo; p.o8;p	c                     t        | j                  | j                  j                  | j                  | j
                  |      S )zJCalculate optimal batch size based on model and device memory constraints.)rZ   r   r   rD   max_num_obj)r   rZ   rA   r   r   rU   )rj   r_  s     rn   r   zBaseTrainer.auto_batch  s2    %**))////#
 	
rv   c                    d\  }}| j                   j                  dk(  rFt        j                  j	                         }|rt        d      j                         j                  dz  S | j                   j                  dk7  rSt        j                  j                         }|r3t        j                  j                  | j                         j                  }|r|dkD  r||z  S dS |dz  S )zJGet accelerator memory utilization in GB or as a fraction of total memory.)r   r   r>   psutilr<   r=   r   i   @)rC   rX   r   r>   driver_allocated_memory
__import__virtual_memorypercentr3   memory_reservedget_device_propertiestotal_memory)rj   fractionmemoryr  s       rn   rD  zBaseTrainer._get_memory  s    ;;u$YY668F!(+::<DDsJJ[[&ZZ//1F

88ERR9AEAI%W1WQVWrv   r  c                 p   |r0d|cxk  rdk  sJ d        J d       | j                  d      |k  ryt        j                          | j                  j                  dk(  rt
        j                  j                          y| j                  j                  dk(  ryt
        j                  j                          y)	zIClear accelerator memory by calling garbage collector and emptying cache.r   r5   z"Threshold must be between 0 and 1.T)ri  Nr>   r=   )	rD  gccollectrC   rX   r   r>   empty_cacher3   )rj   r  s     rn   rI  zBaseTrainer._clear_memory  s    	&Q&L(LL&L(LL&.);


;;u$II!!#[[&JJ""$rv   c                 `    ddl }|j                  | j                        j                  d      S )z0Read results.csv into a dictionary using pandas.r   Nr   )orient)pandasread_csvre   to_dict)rj   pds     rn   read_results_csvzBaseTrainer.read_results_csv  s&    {{488$,,F,;;rv   c                    | j                   j                          | j                   j                         D ]S  \  }t        t	        fd| j
                              s)t        |t        j                        sD|j                          U y)zSet model in training mode.c                     | v S r   r   )fns    rn   r   z*BaseTrainer._model_train.<locals>.<lambda>)  s    AF rv   N)
rZ   r   named_modulesr   filterr   r   r	   BatchNorm2deval)rj   mry  s     @rn   r/  zBaseTrainer._model_train$  sc    

JJ,,. 	DAq6*D,C,CDE*UVXZXfXfJg	rv   c                    ddl }|j                         }t        j                  | j                  | j
                  dt        | j                  j                        j                         | j                  j                  t        t        | j                  j                                     t        | j                        i | j                  d| j                   i| j#                         t%        j&                         j)                         t*        ddd|       |j-                         }| j.                  j1                  |       | j
                  | j                   k(  r| j2                  j1                  |       | j4                  dkD  rJ| j                  | j4                  z  dk(  r-| j6                  d| j                   dz  j1                  |       yyy)	z9Save model training checkpoints with additional metadata.r   Nra   z*AGPL-3.0 (https://ultralytics.com/license)zhttps://docs.ultralytics.com)r*  r`   rZ   r]   updatesr   
train_argstrain_metricstrain_resultsdateversionlicensedocsr*  .pt)ioBytesIOr   rP   r*  r`   r   r]   halfr  r)   r   
state_dictrQ   rA   rI   ra   ru  r   now	isoformatr   getvaluerR   write_bytesrS   rT   rN   )rj   r  bufferserialized_ckpts       rn   rL  zBaseTrainer.save_model,  s^    

 $ 1 1-22488++A(4>>KdKdKfBgh"499o!NDLL!NY4M!N!%!6!6!8 002&G6 !	
$ !//+ 			o.,II!!/2q tzzD4D4D'D'IYY5C00==oN (J rv   c           	      n   	 | j                   j                  dk(  r t        | j                   j                        }nz| j                   j                  j	                  dd      d   dv s| j                   j                  dv r7t        | j                   j                        }d|v r|d   | j                   _        | j                   j                  r!t        j                  d       ddid<   d|d<   S # t        $ r=}t        t        dt        | j                   j                         d	|             |d
}~ww xY w)z
        Get train and validation datasets from data dictionary.

        Returns:
            (dict): A dictionary containing the training/validation/test dataset and category names.
        classifyr   r5   r9   >   ymlr  >   r   posedetectsegment	yaml_filez	Dataset 'u   ' error ❌ Nz)Overriding class names with single class.r   itemr  r  )rA   r   r   r\   rsplitr   r   RuntimeErrorr   r   
single_clsr   r   )rj   r\   r   s      rn   r[   zBaseTrainer.get_datasetO  s	   	fyy~~+(8&&sA.r2oE \ J )8$&%)+%6DIIN 99KKCDKDMDJ  	fv	)DIINN2K1LLYZX[&\]^dee	fs   B3C. .	D478D//D4c                    t        | j                  t        j                  j                        ry| j                  d}}d}t        | j                        j                  d      r%t        | j                        \  }}|j                  }nLt        | j                  j                  t
        t        f      r"t        | j                  j                        \  }}| j                  ||t        dk(        | _        |S )z
        Load, create, or download model for any task.

        Returns:
            (dict): Optional checkpoint to resume training from.
        Nr  r9   )rk   r7   verbose)r   rZ   r   r	   ModulerE   endswithr   r  rA   
pretrainedr   	get_modelr   )rj   rk   r7   r   _s        rn   r   zBaseTrainer.setup_modelj  s     djj%((//2zz4Wtzz?##E*3DJJ?MGT,,C		,,sDk:01E1EFJGQ^^Wdbj^Q
rv   c                    | j                   j                  | j                         t        j                  j
                  j                  | j                  j                         d       | j                   j                  | j                         | j                   j                          | j                  j                          | j                  r&| j                  j                  | j                         yy)zVPerform a single step of the training optimizer with gradient clipping and EMA update.g      $@)max_normN)r   unscale_r   r   r	   utilsclip_grad_norm_rZ   
parametersr.  updater)  r]   r   s    rn   r@  zBaseTrainer.optimizer_step~  s    T^^,&&tzz'<'<'>&N(  "88HHOODJJ' rv   c                     |S )zQAllow custom preprocessing model inputs and ground truths depending on task type.r   )rj   rD   s     rn   r;  zBaseTrainer.preprocess_batch  s    rv   c                     | j                  |       }|j                  d| j                  j                         j	                         j                                }| j                  r| j                  |k  r|| _        ||fS )z
        Run validation on test set using self.validator.

        Returns:
            metrics (dict): Dictionary of validation metrics.
            fitness (float): Fitness score for the validation.
        ra   )rH   poprb   detachr=   numpyr`   )rj   rI   ra   s      rn   rJ  zBaseTrainer.validate  sm     ..&++i$))*:*:*<*@*@*B*H*H*J)JK  D$5$5$? 'Drv   c                     t        d      )z>Get model and raise NotImplementedError for loading cfg files.z3This task trainer doesn't support loading cfg filesNotImplementedError)rj   rk   r7   r  s       rn   r  zBaseTrainer.get_model  s    !"WXXrv   c                     t        d      )zGReturn a NotImplementedError when the get_validator function is called.z1get_validator function not implemented in trainerr  r   s    rn   r   zBaseTrainer.get_validator      !"UVVrv   c                     t        d      )z5Return dataloader derived from torch.data.Dataloader.z2get_dataloader function not implemented in trainerr  )rj   dataset_pathrU   r   r   s        rn   r   zBaseTrainer.get_dataloader  s    !"VWWrv   c                     t        d      )zBuild dataset.z1build_dataset function not implemented in trainerr  )rj   img_pathr   rD   s       rn   build_datasetzBaseTrainer.build_dataset  r  rv   c                     |d|iS dgS )z
        Return a loss dict with labelled training loss items tensor.

        Note:
            This is not needed for classification but necessary for segmentation & detection
        rb   r   )rj   r<  r   s      rn   r   zBaseTrainer.label_loss_items  s     (2'=
#KF8Krv   c                 @    | j                   d   | j                  _        y)z/Set or update model parameters before training.r  N)r\   rZ   r  r   s    rn   r   z BaseTrainer.set_model_attributes  s    99W-

rv   c                      y)z-Build target tensors for training YOLO model.Nr   )rj   predstargetss      rn   build_targetszBaseTrainer.build_targets      rv   c                      y)z-Return a string describing training progress. r   r   s    rn   r5  zBaseTrainer.progress_string  s    rv   c                      y)z+Plot training samples during YOLO training.Nr   )rj   rD   rU  s      rn   rF  z!BaseTrainer.plot_training_samples  r  rv   c                      y)z$Plot training labels for YOLO model.Nr   r   s    rn   r   z BaseTrainer.plot_training_labels  r  rv   c           	      6   t        |j                               t        |j                               }}t        |      dz   }| j                  j                         rdn'd|z  t        ddg|z         z  j                  d      dz   }t        j                         | j                  z
  }t        | j                  dd	
      5 }|j                  |d|z  t        | j                  dz   |g|z         z  j                  d      z   dz          ddd       y# 1 sw Y   yxY w)z$Save training metrics to a CSV file.r@   r  z%s,r*  r#  r}   r  azutf-8)encodingz%.6g,r5   N)r   r   valuesr   re   existsr   rstripr#  r%  openwriter*  )rj   rI   r   valsry  sr\  rx  s           rn   rK  zBaseTrainer.save_metrics  s    ',,.)40@+AdL1((//#B519ugv=NQU=U7V+V*^*^_b*cfj*jIIK$///$((C'2 	^aGGA1udjj1na-@4-G'HHPPQTUUX\\]	^ 	^ 	^s   >ADDc                      y)z"Plot and display metrics visually.Nr   r   s    rn   rN  zBaseTrainer.plot_metrics  r  rv   c                 b    t        |      }|t        j                         d| j                  |<   y)z2Register plots (e.g. to be consumed in callbacks).)r\   	timestampN)r   r#  rJ   )rj   rM   r\   paths       rn   on_plotzBaseTrainer.on_plot  s$    Dz$(tyy{C

4rv   c                    i }| j                   | j                  fD ]  }|j                         s|| j                   u rt        |      }.|| j                  u s=d}t        |||v r|||   ind       t	        j
                  d| d       | j                  j                  | j                  j                  _        | j                  |      | _	        | j                  j                  dd       | j                  d        y)	zHPerform final evaluation and validation for object detection YOLO model.r  N)r  z
Validating z...)rZ   ra   r  )rR   rS   r  r-   r   r   rA   rJ   rH   rI   r  r{   )rj   r   rx  r   s       rn   rM  zBaseTrainer.final_eval  s    DII% 	;Axxz		>*1-D$))^'A#AqDy47|dSKK-s# 6704		DNN''-#'>>>#:DLLL$$Y5&&'9:	;rv   c                    | j                   j                  }|r	 t        |t        t        f      xr t	        |      j                         }t	        |rt        |      n	t                     }t        |      j                   }t        |d   t              s5t	        |d         j                         s| j                   j                  |d<   d}t        |      | _         t        |      x| j                   _        | j                   _        dD ]!  }||v st        | j                   |||          # 	 || _        y|| _        y# t        $ r}t        d      |d}~ww xY w)zCCheck if resume checkpoint exists and update arguments accordingly.r\   T)r   rD   rC   r'  zzResume checkpoint not found. Please pass a valid checkpoint to resume from, i.e. 'yolo train resume model=path/to/last.pt'N)rA   resumer   rE   r   r  r   r$   r   r   r\   r   rZ   setattrr   FileNotFoundError)rj   rl   r  r  rR   	ckpt_argsr   r   s           rn   rB   zBaseTrainer.check_resume  s-   !!#FS$K8RT&\=P=P=R&Jv.n>NO 16;;	!)F"3T:4	RXHYCZCaCaCc(,		If%#I.	58Y>		$))"2 <A I~		1il;< f  'E s   C4D= D= =	EEEc           	         || j                   syd}|j                  dd      dz   }|j                  dd      #| j                  j                  |d          |d   }| j                  ri|j                  d      rX| j                  j                  j                  |d   j                         j                                |d	   | j                  _        |d
kD  sBJ | j                  j                   d| j                   d| j                  j                   d       t        j                  d| j                  j                   d|dz    d| j                   d       | j                  |k  rMt        j                  | j                   d|d    d| j                   d       | xj                  |d   z  c_
        || _        || _        || j                  | j                  j                  z
  kD  r| j!                          yy)z7Resume YOLO training from given epoch and best fitness.Nr  r*  r9   r5   r   r`   r]   r  r   z training to zf epochs is finished, nothing to resume.
Start a new training without resuming, i.e. 'yolo train model=r   zResuming training z from epoch z to z total epochsz has been trained for z epochs. Fine-tuning for z more epochs.)r  rz   r   load_state_dictr]   floatr  r  rA   rZ   rV   r   r   r`   rW   r'  r3  )rj   r   r`   rW   s       rn   r   zBaseTrainer.resume_training
  s   <t{{hhw+a/88K&2NN**4+<=/L88HHLL((e):):)<)G)G)IJ#IDHHQ 	
yy}T[[M :MMQYY__L]]^`	
 	((9kTUoEVVZ[_[f[fZggtuv;;$KK::,4T']OC\]a]h]h\iivw KK4=(K(&$++		(>(>>?))+ @rv   c                 X   t        | j                  j                  d      rd| j                  j                  _        t        | j                  j                  d      rOt	        j
                  d       | j                  j                  j                  t        | j                               yy)z5Update dataloaders to stop using mosaic augmentation.mosaicFr'  zClosing dataloader mosaic)hypN)	r   r   r   r  r   r   r'  r   rA   r   s    rn   r3  z$BaseTrainer._close_dataloader_mosaic%  sy    4$$,,h7/4D%%,4$$,,n=KK34%%22tDII2G >rv   c                    g g g f}t        d t        j                  j                         D              }|dk(  rt	        j
                  t        d       d| j                  j                   d| j                  j                   d       | j                  j                  dd      }	t        d	d
|	z   z  d      }
|dkD  rdnd|
df\  }}}d| j                  _        |j                         D ]|  \  }}|j                  d      D ]b  \  }}|r| d| n|}d|v r|d   j!                  |       *t#        ||      sd|v r|d   j!                  |       O|d   j!                  |       d ~ h d}|D ci c]  }|j%                         | c}j                  |j%                               }|dv r/ t'        t(        |t(        j*                        |d   ||dfd      }nU|dk(  rt)        j,                  |d   ||      }n4|dk(  rt)        j.                  |d   ||d !      }nt1        d"| d#| d$      |j3                  |d   |d%       |j3                  |d   dd%       t	        j
                  t        d       d&t5        |      j6                   d'| d(| d)t9        |d          d*t9        |d          d+| d,t9        |d          d-       |S c c}w ).a  
        Construct an optimizer for the given model.

        Args:
            model (torch.nn.Module): The model for which to build an optimizer.
            name (str, optional): The name of the optimizer to use. If 'auto', the optimizer is selected
                based on the number of iterations.
            lr (float, optional): The learning rate for the optimizer.
            momentum (float, optional): The momentum factor for the optimizer.
            decay (float, optional): The weight decay for the optimizer.
            iterations (float, optional): The number of iterations, which determines the optimizer if
                name is 'auto'.

        Returns:
            (torch.optim.Optimizer): The constructed optimizer.
        c              3   0   K   | ]  \  }}d |v s|  yw)NormNr   )r   r   r  s      rn   r   z.BaseTrainer.build_optimizer.<locals>.<genexpr>?  s     DA!1Ds   autoz
optimizer:z' 'optimizer=auto' found, ignoring 'lr0=z' and 'momentum=zJ' and determining best 'optimizer', 'lr0' and 'momentum' automatically... r  
   {Gz?      i'  )SGDr  ?AdamWr  r  F)recurser   biasr@   logit_scaler5   r   >   r  Adamr  r  NAdamRAdamAdamaxRMSProp>   r  r  r  r  r  g+?)r   betasr   r  )r   r   r  T)r   r   nesterovzOptimizer 'z,' not found in list of available optimizers zX. Request support for addition optimizers at https://github.com/ultralytics/ultralytics.)paramsr   r   z(lr=z, momentum=z) with parameter groups z weight(decay=0.0), z weight(decay=z), z bias(decay=0.0))r   r	   __dict__itemsr   r   r   rA   r   r   r\   rz   r   r9  rz  r   rr   r   lowergetattrr
   r  RMSpropr  r  add_param_grouprX   __name__r   )rj   rZ   rM   r   r   r   r   gbnr  lr_fitmodule_namemodule
param_nameparamfullname
optimizersr   r   s                      rn   r   zBaseTrainer.build_optimizer-  s   " BJD!2!2!4DD6>KKL)* +!!%/?		@R@R?S TWX
 tR(B9B/3F7AE7I!3PWY_adOeD"h'*DII$#(#6#6#8 		'K%+%<%<U%<%K '!
E<Gk]!J<8ZX%aDKK&+}/HaDKK&aDKK&'		' ]
&01	1155djjlC@@8tUZZ81"XW\L]lopIYadrHEIU]		!A$24PI%dV#OPZ| \i i 
 	!!QqT5"IJ!!QqT3"GH%&aY(@(@'AbTU]T^^v1Q4yk-c!A$i[ugSQTUVWXUYQZP[[km	
 ' 2s   .K)r5   )r   )Fr   )NNT)r   r   r   )r   N)Nr   )r  gMbP?r  gh㈵>g     j@)-r  
__module____qualname____doc__r   ro   rE   ru   rx   r{   r   r   r   r  r   r   rD  r  rI  ru  r/  rL  r[   r   r@  r;  rJ  r  r   r   r  r   r   r  r5  rF  r   rK  rN  r  rM  rB   r   r3  r   r   rv   rn   r1   r1   ;   s    0d '$4 B6H/# /+# +3 
$'LX
\6|_'B
X%u %<OF6(( YWXWL.
^D
; @,6H=rv   r1   )Dr	  rl  r   rF   r   r#  r+  r   r   r   r   pathlibr   r  r6  r   r   r   r	   r
   ultralyticsr   ultralytics.cfgr   r   ultralytics.data.utilsr   r   ultralytics.nn.tasksr   r   ultralytics.utilsr   r   r   r   r   r   r   r   r   r   ultralytics.utils.autobatchr   ultralytics.utils.checksr   r   r   r    r!   ultralytics.utils.distr"   r#   ultralytics.utils.filesr$   ultralytics.utils.torch_utilsr%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r1   r   rv   rn   <module>r     s    
  	     (    %  # 1 G N   ? o o D 2   o orv   