
    h45                         d dl mZmZ d dlmZ ddlmZmZ ddlm	Z	 ddl
mZmZ  G d d	e      Z e        G d
 de             Z e        G d de             ZeedZy)    )ABCabstractmethod)deque   )attach_tracertraced   )PagedAttentionCache)RequestStateRequestStatusc                       e Zd ZdZddedefdZedefd       Z	ede
dee   fd	       Zedefd
       Zeddedefd       Zededee
   fd       Zy)	Schedulerz
    Abstract base class for scheduling requests in the continuous batch processor.
    It is expected that cache allocation and scheduling logic will be implemented in subclasses.
    cacheretain_cache_on_finishc                 Z    i | _         i | _        t               | _        || _        || _        y N)active_requestswaiting_requestsr   waiting_requests_orderr   r   )selfr   r   s      s/var/www/html/eduruby.in/venv/lib/python3.12/site-packages/transformers/generation/continuous_batching/scheduler.py__init__zScheduler.__init__   s+    8:9;27'#
&<#    statec                      yz"Add a request to the waiting list.N )r   r   s     r   add_waiting_requestzScheduler.add_waiting_request$        	r   token_budgetreturnc                      y r   r   )r   r    s     r   schedule_batchzScheduler.schedule_batch)   s    r   c                 Z    t        | j                        xs t        | j                        S )z2Check if there are requests ready to be processed.)lenr   r   )r   s    r   has_pending_requestszScheduler.has_pending_requests-   s%     4''(FC0E0E,FFr   
request_idevict_from_cachec                      y)z:Finish processing a request and free its allocated blocks.Nr   r   r'   r(   s      r   finish_requestzScheduler.finish_request2   r   r   c                 T    || j                   v r| j                   |   j                  S g S r   )r   static_outputs)r   r'   s     r   !get_active_request_static_outputsz+Scheduler.get_active_request_static_outputs7   s,    ---''
3BBB	r   N)FT)__name__
__module____qualname____doc__r
   boolr   r   r   r   intlistr#   r   r&   strr+   r.   r   r   r   r   r      s    
=1 =4 =    3 43E   Gd G G     C DI  r   r   c                        e Zd Zddededef fdZedede	fd       Z
 ed	      ded
e	dee   fd       Zedefd       Zed
e	dee   fd       Zeddedefd       Z xZS )FIFOSchedulerr   r   safety_marginc                 4    t         |   ||       || _        y r   )superr   r:   )r   r   r   r:   	__class__s       r   r   zFIFOScheduler.__init__@   s     67*r   r   len_next_tokensc                    |j                         }t        |j                        | j                  j                  z  |z
  }||k  st        |j                        dk(  rf||z
  dz   | j                  j                  z  dz   }| j                  j                  ||j                        }|sy|j                  j                  |       yNr   r	   FTcurrent_lenr%   allocated_blocksr   
block_sizeallocate_blocksr'   extendr   r   r>   rB   	occupancyblocks_needed	allocateds          r   _allocate_blocks_if_neededz(FIFOScheduler._allocate_blocks_if_neededD        '')../$**2G2GG+U	&3u/E/E+F!+K-	9A=$**BWBWW[\\M

22=%BRBRSI"")))4r   prepare_request	span_namer    "request_ids_to_remove_from_waitingc                 J   |j                   t        j                  k(  r|j                  n|j                  }t        |      |k  r|j                   t        j                  k(  rJ|| j                  |j                  <   t        j                  |_         |j                  |j                         y|j                   t        j                  k(  r.t        j                  |_         |j                  |_        g |_        yy|j                   t        j                  k(  rJ|| j                  |j                  <   t        j                  |_         |j                  |j                         n2|j                   t        j                  k(  rt        j                  |_         ||d |_        |d| |_        yz6Prepare a request for processing in the current batch.Nstatusr   SPLIT_PENDING_REMAINDERremaining_prompt_ids
prompt_idsr%   PENDINGr   r'   
PREFILLINGaddPREFILLING_SPLITr   r   r    rP   request_tokenss        r   _prepare_request_for_processingz-FIFOScheduler._prepare_request_for_processingR   G    +0,,-:_:_*_E&&ejeueu 	 ~-||}4449>$$U%5%56,77266u7G7GH!F!FF,77#(#=#= -/* G ||}4449>$$U%5%56,==266u7G7GH!F!FF,==)7)FE&-m|<Er   c                    | j                   r|j                  | j                  v rn| j                  j                  |j                        }|j                  t        |j                        d |_        |j                  |_        |j                  |_        || j                  |j                  <   | j                  j                  |j                         yr   r   r'   r   poprW   r%   full_prompt_idsrC   position_offsetr   r   appendr   r   	old_states      r   r   z!FIFOScheduler.add_waiting_requesto        &&5+;+;t?S?S+S,,001A1ABI$//I4M4M0N0PQE%.%?%?E"$-$=$=E!27e../##**5+;+;<r   r!   c                 :    g }g }g  j                   j                         D ]_  }|j                  t        j                  k(  r|j                  |       |j                  t        j                  k(  sO|j                  |       a  j                  D ]   }|j                   j                  |          " ||z   }t                j                   j                  j                  z  }|D ]  } j                  j                         }||k  }	|	r!r|j                  t        j                  k7  r n j                  ||       t        |j                         }
 j#                  |t        |j                               s%t         j                  j$                        dk(  r nJt&        dt(        ffd       } ||       ||
z  }t&        dt(        f fd       } ||       |dk(  s n t+         j                  D cg c]	  }|vs| c}       _        S c c}w )Nr   r   c                 (    j                  |        y r   re   r   scheduled_requestss    r   _add_to_scheduled_requestsz@FIFOScheduler.schedule_batch.<locals>._add_to_scheduled_requests       "))%0r   c                 x    | j                   }|j                  v rj                  |= j                  |       y y r   r'   r   rZ   r   req_idrP   r   s     r   _remove_from_waiting_requestszCFIFOScheduler.schedule_batch.<locals>._remove_from_waiting_requests   =    ))T222--f56::6B 3r   )r   valuesrT   r   DECODINGre   rU   r   r   setr:   r   
num_blocksget_num_free_blocksr^   r%   rW   rK   _free_blocksr   r   r   )r   r    priority_statessecond_priority_statesr   rs   
candidatessafety_marginsnum_free_blocksoutside_safety_marginrequest_lenrn   rt   rP   rm   s   `            @@r   r#   zFIFOScheduler.schedule_batchz   s   .057))002 	5E||}555&&u-||}DDD&--e4		5 11 	IF"))$*?*?*GH	I %'==
-0U*++djj.C.CC "	E"jj<<>O$3n$D!$);P]PfPf@f00Fhie../K22s5++, tzz../141, 1 1 'u-K'LC\ C C *%0q E"	H ',"&"="=rOqAqVr'
# "! ss   <	HHr'   r(   c                 x    |r8| j                   j                  |       || j                  v r| j                  |= y y y r   r   free_blocksr   r*   s      r   r+   zFIFOScheduler.finish_request   =    JJ"":.T111((4 2 r   )Fg        r/   )r0   r1   r2   r
   r4   floatr   r   r   r5   rK   rx   r7   r^   r   r6   r#   r+   __classcell__)r=   s   @r   r9   r9   >   s    +1 +4 +hm +  s   '(=!=14=Z]^aZb= )=8 = = = ;"3 ;"43E ;" ;"z 5 5 5 5r   r9   c                       e Zd Zededefd       Z ed      dededee   fd       Z	edefd	       Z
eded
ee   fd       Zeddedefd       Zy)PrefillFirstSchedulerr   r>   c                    |j                         }t        |j                        | j                  j                  z  |z
  }||k  st        |j                        dk(  rf||z
  dz   | j                  j                  z  dz   }| j                  j                  ||j                        }|sy|j                  j                  |       yr@   rA   rG   s          r   rK   z0PrefillFirstScheduler._allocate_blocks_if_needed   rL   r   rM   rN   r    rP   c                 J   |j                   t        j                  k(  r|j                  n|j                  }t        |      |k  r|j                   t        j                  k(  rJ|| j                  |j                  <   t        j                  |_         |j                  |j                         y|j                   t        j                  k(  r.t        j                  |_         |j                  |_        g |_        yy|j                   t        j                  k(  rJ|| j                  |j                  <   t        j                  |_         |j                  |j                         n2|j                   t        j                  k(  rt        j                  |_         ||d |_        |d| |_        yrR   rS   r\   s        r   r^   z5PrefillFirstScheduler._prepare_request_for_processing   r_   r   c                    | j                   r|j                  | j                  v rn| j                  j                  |j                        }|j                  t        |j                        d |_        |j                  |_        |j                  |_        || j                  |j                  <   | j                  j                  |j                         yr   ra   rf   s      r   r   z)PrefillFirstScheduler.add_waiting_request   rh   r   r!   c                 r   
 g }g }g  j                   j                         D ]`  }|j                  t        j                  k(  r|j                  |       2|j                  t        j                  k(  sP|j                  |       b  j                  D ]   }|j                   j                  |          " ||z   }t               
|D ]  } j                  ||
       t        |j                        } j                  |t        |j                              s%t         j                  j                        dk(  r nJut         dt"        ffd       } ||       ||z  }t         dt"        f
 fd       }	 |	|       |dk(  s n t%         j                  D cg c]	  }|
vs| c}       _        S c c}w )Nr   r   c                 (    j                  |        y r   rk   rl   s    r   rn   zHPrefillFirstScheduler.schedule_batch.<locals>._add_to_scheduled_requests  ro   r   c                 x    | j                   }|j                  v rj                  |= j                  |       y y r   rq   rr   s     r   rt   zKPrefillFirstScheduler.schedule_batch.<locals>._remove_from_waiting_requests  ru   r   )r   rv   rT   r   rU   re   rw   r   r   rx   r^   r%   rW   rK   r   r{   r   r   r   )r   r    r|   r}   r   rs   r~   r   rn   rt   rP   rm   s   `         @@r   r#   z$PrefillFirstScheduler.schedule_batch   s   .057))002 	5E||}DDD&&u-!7!77&--e4		5 11 	IF"))$*?*?*GH	I %'==
-0U* 	E00Fhie../K22s5++, tzz../141, 1 1 'u-K'LC\ C C *%0q 9	< ',"&"="=rOqAqVr'
# "! ss   	F4"F4r'   r(   c                 x    |r8| j                   j                  |       || j                  v r| j                  |= y y y r   r   r*   s      r   r+   z$PrefillFirstScheduler.finish_request/  r   r   Nr/   )r0   r1   r2   r   r   r5   rK   rx   r7   r^   r   r6   r#   r4   r+   r   r   r   r   r      s     s   '(=!=14=Z]^aZb= )=8 = = = 4"3 4"43E 4" 4"l 5 5 5 5r   r   )fifoprefill_firstN)abcr   r   collectionsr   utils.metricsr   r   r   r
   classesr   r   r   r9   r   SCHEDULER_MAPPINGr   r   r   <module>r      su    $  2 & 0$ $N ~5I ~5 ~5B s5I s5 s5n * r   