
    hЍ                     \   d dl Z d dlmZmZmZ d dlZd dlmc mZ	 d dlmZ ddl
mZ ddlmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZ ddlmZmZ ddlmZm Z  ddl!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1  e*       rd dl2m3Z3  ed       G d dejh                               Z5 G d dejh                        Z6d Z7dEdZ8dejr                  de:dejr                  fd Z;	 dFd!ejh                  d"ejr                  d#ejr                  d$ejr                  d%eejr                     d&e<d'e<d(e%e'   fd)Z=	 	 	 dGd!ejh                  d"ejr                  d#ejr                  d$ejr                  d%eejr                  d*f   d&ee<   d+ee<   d,eejr                     de>ejr                  ejr                  f   fd-Z? e"       Z@e?e@d.<    G d/ d0ejh                        ZA G d1 d2ejh                        ZB G d3 d4ejh                        ZC G d5 d6e      ZDe( G d7 d8e#             ZEe( G d9 d:eE             ZF	 	 	 	 dHd;eejr                  e>ejr                     df   d<ee:   d=ee:   d>e:d%eejr                     deejr                  e:f   fd?ZGe( G d@ dAeEe             ZH G dB dCeeE      ZIg dDZJy)I    N)CallableOptionalUnion)nn   )ACT2FN)CacheDynamicCache)GenerationMixin)use_kernel_forward_from_hub)compile_friendly_flex_attention)create_causal_mask!create_sliding_window_causal_mask) GenericForSequenceClassificationGradientCheckpointingLayer)MoeCausalLMOutputWithPastMoeModelOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)AttentionInterfacePreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupleis_torch_flex_attn_available)deprecate_kwarg)OutputRecordercheck_model_inputs   )
DogeConfig)	BlockMaskRMSNormc                   ,     e Zd Zd fd	Zd Zd Z xZS )DogeRMSNormc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z:
        DogeRMSNorm is equivalent to T5LayerNorm
        N)super__init__r   	Parametertorchonesweightvariance_epsilon)selfhidden_sizeeps	__class__s      d/var/www/html/eduruby.in/venv/lib/python3.12/site-packages/transformers/models/doge/modeling_doge.pyr(   zDogeRMSNorm.__init__6   s1     	ll5::k#:; #    c                 "   |j                   }|j                  t        j                        }|j	                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S )N   T)keepdim)	dtypetor*   float32powmeanrsqrtr-   r,   )r.   hidden_statesinput_dtypevariances       r2   forwardzDogeRMSNorm.forward>   sy    #))%((7 $$Q',,R,>%Ht?T?T4T(UU{{]--k:::r3   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)tupler,   shaper-   )r.   s    r2   
extra_reprzDogeRMSNorm.extra_reprE   s*    ))*+6$2G2G1HIIr3   )gư>)__name__
__module____qualname__r(   rA   rE   __classcell__r1   s   @r2   r%   r%   4   s    $;Jr3   r%   c                   ~     e Zd ZU ej                  ed<   ddef fdZ ej                         e	d               Z
 xZS )DogeRotaryEmbeddinginv_freqconfigc                    t         |           t        |d      rUt        |j                  t
              r;|j                  j                  d|j                  j                  d            | _        nd| _        |j                  | _	        |j                  | _
        || _        t        | j                     | _        | j                  | j                  |      \  }| _        | j                  d|d       | j                   | _        y )Nrope_scaling	rope_typetypedefaultrM   F)
persistent)r'   r(   hasattr
isinstancerP   dictgetrQ   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrN   r   rope_init_fnattention_scalingregister_bufferrM   original_inv_freq)r.   rN   devicerM   r1   s       r2   r(   zDogeRotaryEmbedding.__init__L   s    6>*z&:M:Mt/T#0044[&BUBUBYBYZ`BabDN&DN"("@"@$*$B$B!/?+/+<+<T[[&+Q($(ZeD!%r3   c                 b   | j                   d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        j                  |d      5  |j                         |j                         z  j                  dd      }t        j                  ||fd	      }|j                         | j                  z  }|j                         | j                  z  }	d d d        j	                  |j                   
      	j	                  |j                   
      fS # 1 sw Y   AxY w)Nr   r6   r    mpscpuF)device_typeenabledr5   dim)r8   )rM   floatexpandrD   r9   r`   rV   rR   strr*   autocast	transposecatcosr]   sinr8   )
r.   xposition_idsinv_freq_expandedposition_ids_expandedrd   freqsembrn   ro   s
             r2   rA   zDogeRotaryEmbedding.forward]   sV    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfk^^UC 	5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C		5 vvAGGv$cff177f&;;;	5 	5s    BF%%F.N)rF   rG   rH   r*   Tensor__annotations__r!   r(   no_gradr   rA   rI   rJ   s   @r2   rL   rL   I   s=    ll/z /" U]]_<  <r3   rL   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr6   r5   rf   )rD   r*   rm   )rp   x1x2s      r2   rotate_halfr}   m   sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r3   c                     |j                  |      }|j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer}   )qkrn   ro   rq   unsqueeze_dimq_embedk_embeds           r2   apply_rotary_pos_embr   t   sY    ( --
&C
--
&C3w;q>C/0G3w;q>C/0GGr3   r>   n_repreturnc                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r    N)rD   ri   reshape)r>   r   batchnum_key_value_headsslenhead_dims         r2   	repeat_kvr      so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTr3   modulequerykeyvalueattention_maskscalingdropoutkwargsc                 T   t        || j                        }t        || j                        }	t        j                  ||j	                  dd            |z  }
|#|d d d d d d d |j
                  d   f   }|
|z   }
t        j                  j                  |
dt        j                        j                  |j                        }
t        j                  j                  |
|| j                        }
t        j                  |
|	      }|j	                  dd      j                         }||
fS )Nr5   r   r6   )rg   r8   ptrainingr    )r   num_key_value_groupsr*   matmulrl   rD   r   
functionalsoftmaxr:   r9   r8   r   r   
contiguous)r   r   r   r   r   r   r   r   
key_statesvalue_statesattn_weightscausal_maskattn_outputs                r2   eager_attention_forwardr      s    3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL!$Q1.D
0@0@0D.D%DE#k1==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$r3   r"   softcap	head_maskc           
      2   d }	d t        |t              r|}	n|d d d d d d d |j                  d   f   fd}
t        ||||
|	d|d      \  }}|j	                  |j
                        }|j                  dd      j                         }||fS )Nr   c                     t        j                  | z        z  } | |   |   |   |   z   } | |   |   d   d   z   } | S )Nr   )r*   tanh)score	batch_idxhead_idxq_idxkv_idxr   r   r   s        r2   	score_modz)flex_attention_forward.<locals>.score_mod   sm    ejj99E"K	28<UCFKKE Ii0:1=a@@Er3   T)r   
block_mask
enable_gqascale
return_lser    r5   )rV   r"   rD   r   r9   r8   rl   r   )r   r   r   r   r   r   r   r   r   r   r   r   attention_weightsr   s         ``     @r2   flex_attention_forwardr      s     JK.),#
$!!Q?SYYr]?":; &E &"K" *,,U[[9''1-88:K)))r3   doge_flex_attentionc                       e Zd Zddedee   f fdZ eddd      	 	 	 ddej                  d	e
ej                  ej                  f   d
eej                     dee   deej                     de
ej                  eej                     ee
ej                        f   fd       Z	 	 ddej                  dej                  ded
eej                     fdZ xZS )DogeAttentionrN   	layer_idxc                    t         |           || _        || _        t	        |d|j
                  |j                  z        | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        |j                  | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j&                  t)        j*                  |j                              | _        t        j                  |j                  | j                  z  |j                  |j                        | _        t        j                  |j                  | j                  z  |j
                  |j                        | _        t3        | j                  |j4                        | _        t3        | j                  |j4                        | _        y )Nr   g      ࿩biasr0   )r'   r(   rN   r   getattrr/   num_attention_headsr   r   r   r   attention_dropoutkeep_window_sizer   Linearattention_biasq_projk_projv_projr)   r*   zerosAdt_projo_projr%   rms_norm_epsq_normk_normr.   rN   r   r1   s      r2   r(   zDogeAttention.__init__   s   "
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9 & 7 7ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ekk&*D*DEFyy&&68R8RY_YnYn
 ii&&68J8JQWQfQf
 "$--V5H5HI!$--V5H5HIr3   past_key_valuepast_key_values4.58new_nameversionr>   position_embeddingsr   cache_positionr   c                    |j                   d d }g |d| j                  }| j                  | j                  |      j	                  |            j                  dd      }	| j                  | j                  |      j	                  |            j                  dd      }
| j                  |      j	                  |      j                  dd      }|\  }}t        |	|
||      \  }	}
|'|||d}|j                  |
|| j                  |      \  }
}| j                  |j                  dd      j                  |j                   d   |j                   d   d            }t        j                  | j                   t#        j$                  |      z        j                  dd      }| j'                  ||| j(                  |      }t+        || j,                        }t.        }| j0                  j2                  dk7  rt4        | j0                  j2                     } || |	|
|f|| j6                  sd	n| j8                  | j:                  d
|\  }} |j                  g |d j=                         }| j?                  |      }||fS )Nr6   r    r5   )ro   rn   r   r   r   )r>   	dt_statesr   r   eager        )r   r   r   ) rD   r   r   r   viewrl   r   r   r   r   updater   r   r   r*   expr   Fsoftplusprepare_dynamic_maskr   r   r   r   rN   _attn_implementationALL_ATTENTION_FUNCTIONSr   r   r   r   r   )r.   r>   r   r   r   r   r   input_shapehidden_shapequery_statesr   r   rn   ro   cache_kwargsr   	attn_maskattention_interfacer   r   s                       r2   rA   zDogeAttention.forward	  so    $))#2.88b8$--8{{4;;}#=#B#B<#PQ[[\]_`a[[]!;!@!@!NOYYZ[]^_
{{=166|DNNqRST&S#7jRUWZ#[ j&#&snUL'6'='=j,X\XfXfht'u$J LL""1a(001C1CA1FHZHZ[]H^`bc
	 IIdffqzz)'<<=GGBO	--'!22)	 . 
	 i)B)BC	(?;;++w6"9$++:Z:Z"[$7		%

 %#}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((r3   r   r   c           	         t        j                  |j                        j                  }|j                  }|dddddddf   j	                  dd|j
                  d   d      }|t        |t              s|j                  t         j                  k(  rC|j                  }t        j                  |t        j                  d|j                  |      |      }|j                  |ddddddd|j
                  d   f   dk7  |      }|j
                  d   |kD  rnt        j                  |||j                        }t        j                  ||ddd	
      j                  }	|j!                  d|	d      }|j                  |dk(  |      }|S )a8  
        The core idea of DMA is to calculate the dynamic attention mask to mask the tokens that should be masked, so as to form sparse attention.

        Combine `dt_states` with `attention_mask` to generate the final `attn_mask`.

        Args:
            hidden_states (`torch.Tensor`): The input hidden_states, used to determine the minimum value of the current input precision.
            dt_states (`torch.Tensor`): dt_states of shape `(batch_size, num_heads, key_sequence_length)`.
            keep_window_size (`int`): The window size of tokens that are not dynamically masked, and dynamic masking is only performed when the sequence length exceeds this value.
            attention_mask (`torch.Tensor`, *optional*): attention mask of shape `(batch_size, 1, query_sequence_length, key_sequence_length)`.
        Nr6   r    r   )r`   r8   r   r8   r`   TF)rg   largestsorted      ?)r*   finfor8   minri   rD   rV   r"   boolwheretensorr`   masked_fill
zeros_liketopkindicesscatter)
r.   r>   r   r   r   	min_dtyper8   r   active_masktopk_indicess
             r2   r   z"DogeAttention.prepare_dynamic_maskB  sg   $ KK 3 3488	##aD!m,33M''*B
	 %j.S##uzz1%++!&"ELL^=R=RZ_$`bk" "--nQ1F[	XZH[F[=[.\`a.aclmI??2!11**9E)JZJZ[K ::i1ArSW`efnnL%--b,DK!--kS.@)LIr3   rv   NNN)i   N)rF   rG   rH   r!   r   intr(   r   r*   rw   rC   r	   
LongTensorrA   r   rI   rJ   s   @r2   r   r      s   Jz Jhsm J< %0A6R
 26+/596)||6) #5<<#=>6) !.	6)
 "%6) !!1!126) 
u||Xell3XeELL>Q5RR	S6) S6)x !%15#||# <<# 	#
 !.#r3   r   c                   $     e Zd Z fdZd Z xZS )DogeMLPc                    t         |           || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  |j                        | _        t        j                  | j                  | j                  |j                        | _	        t        j                  | j                  | j                  |j                        | _
        t        |j                     | _        y )Nr   )r'   r(   rN   r/   intermediate_sizer   r   mlp_bias	gate_projup_proj	down_projr   
hidden_actact_fnr.   rN   r1   s     r2   r(   zDogeMLP.__init__i  s    !--!'!9!94#3#3T5K5KRXRaRabyy!1!143I3IPVP_P_`4#9#94;K;KRXRaRabV../r3   c                     | j                  | j                  | j                  |            | j                  |      z        }|S rv   )r  r	  r  r  )r.   rp   r  s      r2   rA   zDogeMLP.forwards  s6    NN4;;t~~a/@#ADLLQRO#ST	r3   )rF   rG   rH   r(   rA   rI   rJ   s   @r2   r  r  h  s    0r3   r  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )	DogeCDMoErN   c                    t         |           |j                  | _        |j                  | _        t        |j
                     | _        |j                  | _        t        j                  t        j                  | j                              | _        |j                  | _        |j                  | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  dz  d      | _        t        j,                  | j                  | j                        | _        t        j,                  | j                  | j                        | _        y )Nr   r5   F)r'   r(   r/   r  r   r  r	  num_expertsmathfloorsqrtnum_keysnum_experts_per_toktop_knorm_topk_probr   r   r  r  r  r  router_gate	Embedding
down_embedup_embedr
  s     r2   r(   zDogeCDMoE.__init__y  s_   !--!'!9!9V../!--

499T-=-=#>?//
$33 4#3#3T5K5KRXRaRabyy!1!143I3IPVP_P_`4#9#94;K;KRXRaRab 99T%5%5t}}q7HuU ,,t'7'79I9IJT%5%5t7G7GHr3   r>   r   c                    |j                   \  }}}| j                  |      j                  d||z  d      }|j                  | j                  d      \  \  }}\  }	}
|j                  d      |j                  d      z   }|	j                  d      | j                  z  |
j                  d      z   } |j                  g |j                   d d d } |j                  g |j                   d d d }|j                  | j                  d      \  }}|j                  d|      }t        j                  |d      }| j                  r||j                  dd      z  }| j                  |      }| j                  |      }t        j                  ||j                  ||z  dd            j                  ||z  d      }| j!                  |      |z  }t        j                  |j                  ||z  dd      |      j                  ||d      }| j#                  | j!                  | j%                  |            | j'                  |      z        }||z   }||fS )Nr5   r6   rf   r   T)rg   r7   r    )rD   r  r   r   r  r   r  gatherr   r   r  sumr  r  r*   r   r	  r  r  r  )r.   r>   r   bszseq_len_router_logitsscores_xscores_y	indices_x	indices_y
all_scoresall_indicesscoresposition_indicesr   routing_weightsr  r  experts_weightsexperts_statess                        r2   rA   zDogeCDMoE.forward  sF   
 (--Wa ((7<<QgrR 8E7I7I$--]_7I7`484y)''+h.@.@.DD
))"-=	@S@STV@WW$Z__@j&6&6s&;@R@
&k&&C(9(9#2(>CC#-??4::2?#F  $$R)9:))F322r42HHO __W-
==),,z=3E3EcGmUWYZ3[\aabehoboqst++o6Ho&:&:3=!R&PRZ[``adfmoqrt{{4>>-3P'QTXT`T`anTo'op%6m++r3   )	rF   rG   rH   r!   r(   r*   rw   rA   rI   rJ   s   @r2   r  r  x  s0    Iz I.,||, 
	,r3   r  c                       e Zd Zddedee   f fdZ eddd      	 	 	 	 	 ddej                  d	e
ej                  ej                  f   d
eej                     deej                     dee
ej                        dee   deej                     dee   de
ej                  ee
ej                  ej                  f      f   fd       Z xZS )DogeDecoderLayerrN   r   c                 *   t         |           |j                  | _        t        |j                  |j
                        | _        t        ||      | _        t        j                  t        j                  |j                              | _        t        |j                  |j
                        | _        |j                  st!        |      n
t#        |      | _        t        j                  t        j                  |j                              | _        y )Nr   )rN   r   )r'   r(   hidden_dropoutr%   r/   r   input_layernormr   	self_attnr   r)   r*   r+   input_residualpost_attention_layernormis_moer  r  mlppost_attention_residualr   s      r2   r(   zDogeDecoderLayer.__init__  s    $33*6+=+=6CVCVW&f	J ll5::f6H6H+IJ(3F4F4FFL_L_(`%*0--76?Yv=N')||EJJv?Q?Q4R'S$r3   r   r   r   r   r>   r   r   rq   	use_cacher   r   r   c                    |}	| j                  |      } | j                  d|||||||d|\  }}
t        j                  || j                  | j
                        }| j                  |	z  |z   }|}	| j                  |      }| j                  |      }t        j                  || j                  | j
                        }| j                  |	z  |z   }|S )N)r>   r   r   rq   r   r8  r   r    )
r1  r2  r   r   r0  r   r3  r4  r6  r7  )r.   r>   r   r   rq   r   r8  r   r   residualself_attn_weightss              r2   rA   zDogeDecoderLayer.forward  s     !,,];+94>> 	,
' 3)%+)	,
 	,
(( 		-43F3FQUQ^Q^_++h6F !55mD/		-43F3FQUQ^Q^_44x?-Or3   rv   )NNNFN)rF   rG   rH   r!   r   r   r(   r   r*   rw   rC   r   r   r   r   FloatTensorrA   rI   rJ   s   @r2   r.  r.    s   
Tz 
Thsm 
T %0A6R
 26379=$)59"||" #5<<#=>" !.	"
 u//0" "%"56" D>" !!1!12" +," 
u  (51B1BEDUDU1U+V"WW	X" S"r3   r.  c                   n     e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZ eed      eed	Z fd
Z xZS )DogePreTrainedModelrN   modelTr.  r   Fr    )index)r!  r>   
attentionsc                    t         |   |       t        |t              r2t	        |d      r%|j
                  j                  j                          yyt        |t              rdt	        |d      r%|j                  j                  j                  d       t	        |d      r&|j                  j                  j                  d       yyy)zInitialize the weightsr   r3  r   r7  N)r'   _init_weightsrV   r   rU   r   datazero_r.  r3  fill_r7  )r.   r   r1   s     r2   rD  z!DogePreTrainedModel._init_weights  s    f%fm,vs###% $ 01v/0%%**005v89..3399#> : 2r3   )rF   rG   rH   r!   rx   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backendr   r  r.  r   _can_record_outputsrD  rI   rJ   s   @r2   r?  r?    sf    &*#+,#4"5 N""&'	;)#
? 
?r3   r?  c                       e Zd Zdef fdZee	 	 	 	 	 	 	 ddeej                     deej                     deej                     dee   deej                     dee   d	eej                     d
ee   defd              Z xZS )	DogeModelrN   c           	         t         |   |       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                        | _        t#        |      | _        d| _        | j)                          y c c}w )Nr   rN   F)r'   r(   pad_token_idpadding_idx
vocab_sizer   r  r/   embed_tokens
ModuleListrangenum_hidden_layersr.  layersr%   r   normrL   
rotary_embgradient_checkpointing	post_initr   s      r2   r(   zDogeModel.__init__  s     !.. ++LL):):F<N<NPTP`P`ammBGH`H`BabYfi0b
   2 28K8KL	-V<&+# 	 cs   D	input_idsr   rq   r   inputs_embedsr8  r   r   r   c                    |d u |d uz  rt        d      |r|t        | j                        }|| j                  |      }|F||j	                         nd}	t        j                  |	|	|j                  d   z   |j                        }||j                  d      }| j                  j                  t        nt        }
 |
| j                  |||||      }|}| j                  ||      }| j                  d | j                  j                   D ]  } ||f||||||d|} | j!                  |      }t#        ||      S )	Nz:You must specify exactly one of input_ids or inputs_embedsrU  r   r    )r`   )rN   input_embedsr   r   r   rq   )r   r   rq   r   r8  r   )last_hidden_stater   )
ValueErrorr
   rN   rY  get_seq_lengthr*   arangerD   r`   r   sliding_windowr   r   r_  r]  r\  r^  r   )r.   rb  r   rq   r   rc  r8  r   r   past_seen_tokensmask_functionr   r>   r   decoder_layers                  r2   rA   zDogeModel.forward  sx    -t";<YZZ0*$++>O  --i8M!CRC^==?de"\\ "2]5H5H5K"KTaThThN )33A6L.2kk.H.H.P*Vw#;;&))+%
 & #oom\J![[)H4;;+H+HI 
	M)	$7*) /#-	 	M
	 		-0%++
 	
r3   )NNNNNNN)rF   rG   rH   r!   r(   r   r   r   r*   r   rw   r	   r=  r   r   r   r   rA   rI   rJ   s   @r2   rS  rS    s    z    151537+/59$(59<
E,,-<
 !.<
 u//0	<

 "%<
   1 12<
 D><
 !!1!12<
 +,<
 
 <
  <
r3   rS  gate_logitsr  r  r  c                    | t        | t              sy| d   j                  }| d   j                  }g }g }| D ]  }	|	j	                  |      }	|	j                  |d      \  \  }
}\  }}|
j                  d      |j                  d      z   }|j                  d      |z  |j                  d      z   } |j                  g |j                  dd d } |j                  g |j                  dd d }|j                  |d      \  }}|j                  d|      }t        j                  |d      }|j                  |       |j                  |        t        j                  |d      }t        j                  |d      }|}|j                  d      }t        j                  |||      }t        j                   |||      }|j#                  d||      |j                  d   z  }t        j$                  |d      }nD|j                  \  }}t'        |       }|ddddddf   j)                  ||||f      j+                  d      j	                  |      }|j                  d      |j-                            }t        j                  |||      }t        j                   |||      }|j#                  d||      t        j.                  |      z  }|ddddddf   j)                  ||||f      j+                  d|      j	                  |      }t        j.                  ||z  d      t        j.                  |d      z  }t        j.                  ||z        }||z  S )a  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `router_gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [2, batch_size * sequence_length, num_keys].
        num_experts:
            Number of experts
        num_keys:
            Number of keys
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   r6   rf   r   r   )rV   rC   r8   r`   r9   r   r   r   rD   r  r   r   appendr*   rm   r   	ones_likescatter_add_r<   lenri   r   r   r  )rn  r  r  r  r   compute_dtypecompute_deviceall_expert_indicesall_routing_weightslayer_gate_logitsr"  r#  r$  r%  r&  r'  r   r)  expert_indicesr*  tokens_per_expertpadrouter_prob_per_expert
batch_sizesequence_lengthr\  expert_attention_mask router_per_expert_attention_maskoverall_losss                                r2   load_balancing_loss_funcr  V  s{   @ *[%"@N((M ^**N( 4-00@7H7M7Mh\^7M7_484y)''+h.@.@.DD
))"-89;N;Nr;RR$Z__@j&6&6s&;@R@
&k&&C(9(9#2(>CC(ooeo<$++B0@A))JB7!!.1""?3!4" #51=))$7Q?/44R8!KK=Q_`oo0n]-::1>PRUVYkYqYqrsYtt "',?Q!G&4&:&:#
O, 4At+,V&
OUKLWR[R	 	 044R89N9S9S9UV "KK=Q_`oo0n]-::1>PRUVY^YbYb!Z
 
 4At+,V&
O[QRWR%R	 	) "'+>Aa+agh!ilqlulu,!m
 "
 99.1GGHL+%%r3   c                       e Zd ZdgZddiZddgdgfiZ fdZee	 	 	 	 	 	 	 	 	 	 dde	e
j                     de	e
j                     d	e	e
j                     d
e	ee
j                        de	e
j                     de	e
j                     de	e   de	e
j                     deee
j                  f   de	e   dee   defd              Z xZS )DogeForCausalLMzlm_head.weightlm_headcolwise_repr>   logitsc                 N   t         |   |       t        |      | _        |j                  | _        t        j                  |j                  |j                  d      | _        |j                  | _	        |j                  | _
        |j                  | _        | j                          y )NFr   )r'   r(   rS  r@  rX  r   r   r/   r  router_aux_loss_coefr  r  ra  r
  s     r2   r(   zDogeForCausalLM.__init__  s     v&
 ++yy!3!3V5F5FUS$*$?$?!!--#)#=#=  	r3   rb  r   rq   r   rc  labelsr8  r   logits_to_keepoutput_router_logitsr   r   c                    |
|
n| j                   j                  }
 | j                  d|||||||d|}|j                  }t	        |	t
              rt        |	 d      n|	}| j                  |dd|ddf         }d}| | j                  ||| j                  fi |}d}|
rt        |j                  | j                  t        j                  t        j                  | j                              | j                   |      }|+|| j"                  |j%                  |j&                        z  z  }t)        ||||j*                  |j,                  |j.                  |j                        S )ah  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, DogeForCausalLM

        >>> model = DogeForCausalLM.from_pretrained("SmallDoge/Doge-320M")
        >>> tokenizer = AutoTokenizer.from_pretrained("SmallDoge/Doge-320M")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)rb  r   rq   r   rc  r8  r   )lossaux_lossr  r   r>   rB  r!  r:  )rN   r  r@  rf  rV   r   slicer  loss_functionrX  r  r!  r  r  r  r  r  r  r9   r`   r   r   r>   rB  )r.   rb  r   rq   r   rc  r  r8  r   r  r  r   outputsr>   slice_indicesr  r  r  s                     r2   rA   zDogeForCausalLM.forward  sp   N %9$D $++JjJj 	
 +5$** 	+
)%+')	+
 	+
  118B>SV8W~ot4]kmA}a,?@A%4%%ffdooPPD/%%  

499T%5%567((H !11HKK4LLL(#33!//))!//
 	
r3   )
NNNNNNNNr   N)rF   rG   rH   _tied_weights_keys_tp_plan_pp_planr(   r   r   r   r*   r   rw   listr=  r   r   r   r   r   r   rA   rI   rJ   s   @r2   r  r    s[   *+=)H_-z:;H
  151537=A59-1$(5934/3Q
E,,-Q
 !.Q
 u//0	Q

 "$u'8'8"9:Q
   1 12Q
 ))*Q
 D>Q
 !!1!12Q
 c5<</0Q
 'tnQ
 +,Q
 
#Q
  Q
r3   r  c                       e Zd Zy)DogeForSequenceClassificationN)rF   rG   rH   r:  r3   r2   r  r  (  s    r3   r  )r  rS  r?  r  )Nr    )r   r   )NNr5   N)Kr  typingr   r   r   r*   torch.nn.functionalr   r   r   activationsr   cache_utilsr	   r
   
generationr   integrationsr   integrations.flex_attentionr   masking_utilsr   r   modeling_layersr   r   modeling_outputsr   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   utils.deprecationr   utils.genericr   r   configuration_doger!   !torch.nn.attention.flex_attentionr"   Moduler%   rL   r}   r   rw   r   r   rh   r   rC   r   r   r   r  r  r.  r?  rS  r  r  r  __all__r:  r3   r2   <module>r     s6  0  , ,     ! . ) 7 J R [ Q K A & g g 0 ? *  !; Y'J")) J (J(!<")) !<H(6	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 U\\*% % % '(%@  $#(,.*II.*<<.* 
.* <<	.*
 %,,34.* e_.* e_.* %.* 5<<%&.*b -. 1G - .{BII {|bii  6,		 6,r01 0f ?/ ? ?< O
# O
 O
h "&"-1g&u||U5<<%8$>?g&#g& smg& 	g&
 U\\*g& 5<<g&T d
)? d
 d
N	$DFY 	 cr3   