
    h                     0   d Z ddlZddlmZ ddlmZmZmZ ddl	Z	ddl
Z	ddl	mZ ddlmZ ddlmZ dd	lmZmZmZmZ dd
lmZmZ ddlmZ ddlmZmZ ddlmZmZm Z m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z'  e!jP                  e)      Z* G d dejV                        Z, G d dejV                        Z-	 d?dejV                  de	j\                  de	j\                  de	j\                  dee	j\                     de/de/fdZ0 G d dejV                        Z1 G d d ejV                        Z2 G d! d"ejV                        Z3 G d# d$ejV                        Z4 G d% d&ejV                        Z5 G d' d(e      Z6 G d) d*ejV                        Z7e  G d+ d,e             Z8e  G d- d.e8             Z9 G d/ d0ejV                        Z: e d12       G d3 d4e8             Z; e d52       G d6 d7e8             Z<e e d82       G d9 d:e                    Z= e d;2       G d< d=e8             Z>g d>Z?y)@zPyTorch DeiT model.    N)	dataclass)CallableOptionalUnion)nn   )ACT2FN)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutputMaskedImageModelingOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack) find_pruneable_heads_and_indicesprune_linear_layer)ModelOutputTransformersKwargsauto_docstringlogging	torch_int)can_return_tuplecheck_model_inputs   )
DeiTConfigc            	            e Zd ZdZddededdf fdZdej                  de	d	e	dej                  fd
Z
	 	 ddej                  deej                     dedej                  fdZ xZS )DeiTEmbeddingszv
    Construct the CLS token, distillation token, position and patch embeddings. Optionally, also the mask token.
    configuse_mask_tokenreturnNc                    t         |           t        j                  t	        j
                  dd|j                              | _        t        j                  t	        j
                  dd|j                              | _        |r4t        j                  t	        j
                  dd|j                              nd | _	        t        |      | _        | j                  j                  }t        j                  t	        j
                  d|dz   |j                              | _        t        j                  |j                        | _        |j"                  | _        y )Nr      )super__init__r   	Parametertorchzeroshidden_size	cls_tokendistillation_token
mask_tokenDeiTPatchEmbeddingspatch_embeddingsnum_patchesposition_embeddingsDropouthidden_dropout_probdropout
patch_size)selfr   r    r/   	__class__s       d/var/www/html/eduruby.in/venv/lib/python3.12/site-packages/transformers/models/deit/modeling_deit.pyr%   zDeiTEmbeddings.__init__1   s    ekk!Q8J8J&KL"$,,u{{1aASAS/T"UQ_",,u{{1a9K9K'LMei 3F ;++77#%<<A{QPVPbPb0c#d zz&"<"<= ++    
embeddingsheightwidthc                    |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                         s||k(  r||k(  r| j                  S | j                  ddddf   }| j                  ddddf   }|j                   d   }|| j
                  z  }	|| j
                  z  }
t        |dz        }|j                  d|||      }|j                  dddd      }t        j                  j                  ||	|
fdd	
      }|j                  dddd      j                  dd|      }t        j                  ||fd      S )a  
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing and 2 class embeddings.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   r#   N      ?r   r   bicubicF)sizemodealign_cornersdim)shaper0   r'   jit
is_tracingr4   r   reshapepermuter   
functionalinterpolateviewcat)r5   r9   r:   r;   r/   num_positionsclass_and_dist_pos_embedpatch_pos_embedrD   
new_height	new_widthsqrt_num_positionss               r7   interpolate_pos_encodingz'DeiTEmbeddings.interpolate_pos_encoding=   sb    !&&q)A-0066q9A= yy##%+*F6UZ?+++#'#;#;ArrE#B 221ab59r"t.
T__,	&}c'9:)11!5GI[]`a)11!Q1=--33i(	 4 
 *11!Q1=BB1b#Nyy2OD!LLr8   pixel_valuesbool_masked_posrT   c                 "   |j                   \  }}}}| j                  |      }|j                         \  }}	}|K| j                  j	                  ||	d      }
|j                  d      j                  |
      }|d|z
  z  |
|z  z   }| j                  j	                  |dd      }| j                  j	                  |dd      }t        j                  |||fd      }| j                  }|r| j                  |||      }||z   }| j                  |      }|S )Nr=         ?r   rC   )rE   r.   r@   r,   expand	unsqueezetype_asr*   r+   r'   rM   r0   rT   r3   )r5   rU   rV   rT   _r:   r;   r9   
batch_size
seq_lengthmask_tokensmask
cls_tokensdistillation_tokensposition_embeddings                  r7   forwardzDeiTEmbeddings.forwarde   s    +001fe**<8
$.OO$5!
J&//00ZLK",,R088ED#sTz2[45GGJ^^**:r2>
"55<<ZRPYY
,?LRST
!55#!%!>!>z6SX!Y"44
\\*-
r8   )FNF)__name__
__module____qualname____doc__r   boolr%   r'   TensorintrT   r   
BoolTensorrd   __classcell__r6   s   @r7   r   r   ,   s    
,z 
,4 
,D 
,&M5<< &M &MUX &M]b]i]i &MV 7;).	ll "%"2"23 #'	
 
r8   r   c                   Z     e Zd ZdZ fdZdej                  dej                  fdZ xZS )r-   z
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                    t         |           |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                        r|n||f}t        |t        j                  j                        r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _        || _        || _
        t        j                  ||||      | _        y )Nr   r   )kernel_sizestride)r$   r%   
image_sizer4   num_channelsr)   
isinstancecollectionsabcIterabler/   r   Conv2d
projection)r5   r   rt   r4   ru   r)   r/   r6   s          r7   r%   zDeiTPatchEmbeddings.__init__   s    !'!2!2F4E4EJ
$*$7$79K9Kk#-j+//:R:R#SZZdfpYq
#-j+//:R:R#SZZdfpYq
!!}
15*Q-:VW=:XY$$(&))L+:^hir8   rU   r!   c                     |j                   \  }}}}|| j                  k7  rt        d      | j                  |      j	                  d      j                  dd      }|S )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.r#   r   )rE   ru   
ValueErrorr{   flatten	transpose)r5   rU   r]   ru   r:   r;   xs          r7   rd   zDeiTPatchEmbeddings.forward   sa    2>2D2D/
L&%4,,,w  OOL)11!4>>q!Dr8   )	rf   rg   rh   ri   r%   r'   rk   rd   rn   ro   s   @r7   r-   r-      s)    jELL U\\ r8   r-   modulequerykeyvalueattention_maskscalingr3   c                    t        j                  ||j                  dd            |z  }t        j                  j                  |dt         j                        j                  |j                        }t        j                  j                  ||| j                        }|||z  }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )Nr=   )rD   dtype)ptrainingr   r#   )r'   matmulr   r   rJ   softmaxfloat32tor   r3   r   
contiguous)
r   r   r   r   r   r   r3   kwargsattn_weightsattn_outputs
             r7   eager_attention_forwardr      s     <<s}}R'<=GL ==((2U]](SVVW\WbWbcL ==((6??([L !#n4,,|U3K''1-88:K$$r8   c            	            e Zd Zdef fdZ	 ddej                  deej                     deej                  ej                  f   fdZ	 xZ
S )DeiTSelfAttentionr   c                 2   t         |           |j                  |j                  z  dk7  r2t	        |d      s&t        d|j                   d|j                   d      || _        |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _	        |j                  | _        | j                  dz  | _        d| _        t        j                  |j                  | j                  |j                         | _        t        j                  |j                  | j                  |j                         | _        t        j                  |j                  | j                  |j                         | _        y )	Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads .g      F)bias)r$   r%   r)   num_attention_headshasattrr}   r   rl   attention_head_sizeall_head_sizeattention_probs_dropout_probdropout_probr   	is_causalr   Linearqkv_biasr   r   r   r5   r   r6   s     r7   r%   zDeiTSelfAttention.__init__   sF    : ::a?PVXhHi"6#5#5"6 7334A7 
 #)#=#= #&v'9'9F<V<V'V#W !558P8PP"??//5YYv1143E3EFOO\
99V//1C1C&//ZYYv1143E3EFOO\
r8   hidden_states	head_maskr!   c           
         |j                   d   }|d| j                  | j                  f} | j                  |      j                  | j                  dd      } | j                  |      j                  | j                  dd      } | j                  |      j                  | j                  dd      }t        }| j                  j                  dk7  rt        | j                  j                     } || ||||| j                  | j                  | j                  sdn| j                        \  }	}
|	j!                         d d | j"                  fz   }|	j%                  |      }	|	|
fS )	Nr   r=   r   r#   eager        )r   r   r3   r   )rE   r   r   r   rL   r   r   r   r   r   _attn_implementationr   r   r   r   r   r@   r   rH   )r5   r   r   r]   	new_shape	key_layervalue_layerquery_layerattention_interfacecontext_layerattention_probsnew_context_layer_shapes               r7   rd   zDeiTSelfAttention.forward   sR    #((+
D$<$<d>V>VV	0DHH]+00)<FFq!L	4djj/44i@JJ1aP4djj/44i@JJ1aP(?;;++w6"9$++:Z:Z"[)<nnLL#}}C$2C2C	*
& #0"4"4"6s";t?Q?Q>S"S%--.EFo--r8   N)rf   rg   rh   r   r%   r'   rk   r   tuplerd   rn   ro   s   @r7   r   r      sT    ]z ]* PT."\\.6>u||6L.	u||U\\)	*.r8   r   c                   x     e Zd ZdZdef fdZdej                  dej                  dej                  fdZ xZ	S )DeiTSelfOutputz
    The residual connection is defined in DeiTLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    r   c                     t         |           t        j                  |j                  |j                        | _        t        j                  |j                        | _        y r   )	r$   r%   r   r   r)   denser1   r2   r3   r   s     r7   r%   zDeiTSelfOutput.__init__   sB    YYv1163E3EF
zz&"<"<=r8   r   input_tensorr!   c                 J    | j                  |      }| j                  |      }|S r   r   r3   r5   r   r   s      r7   rd   zDeiTSelfOutput.forward  s$    

=1]3r8   )
rf   rg   rh   ri   r   r%   r'   rk   rd   rn   ro   s   @r7   r   r      s=    
>z >
U\\  RWR^R^ r8   r   c                        e Zd Zdef fdZdee   fdZd	dej                  de
ej                     dej                  fdZ xZS )
DeiTAttentionr   c                     t         |           t        |      | _        t	        |      | _        t               | _        y r   )r$   r%   r   	attentionr   outputsetpruned_headsr   s     r7   r%   zDeiTAttention.__init__  s0    *62$V,Er8   headsc                 >   t        |      dk(  ry t        || j                  j                  | j                  j                  | j
                        \  }}t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _	        t        | j                  j                  |d      | j                  _        | j                  j                  t        |      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |      | _        y )Nr   r   rC   )lenr   r   r   r   r   r   r   r   r   r   r   r   union)r5   r   indexs      r7   prune_headszDeiTAttention.prune_heads  s   u:?74>>55t~~7Y7Y[_[l[l
u
  2$..2F2FN/0B0BEJ1$..2F2FN.t{{/@/@%QO .2^^-O-ORUV[R\-\*'+~~'I'IDNNLnLn'n$ --33E:r8   r   r   r!   c                 T    | j                  ||      \  }}| j                  ||      }|S r   )r   r   )r5   r   r   self_attn_outputr\   r   s         r7   rd   zDeiTAttention.forward%  s.    "nn]IF!-}=r8   r   )rf   rg   rh   r   r%   r   rl   r   r'   rk   r   rd   rn   ro   s   @r7   r   r     sM    "z ";S ;$U\\ hu||>T `e`l`l r8   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )DeiTIntermediater   c                    t         |           t        j                  |j                  |j
                        | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y r   )r$   r%   r   r   r)   intermediate_sizer   rv   
hidden_actstrr	   intermediate_act_fnr   s     r7   r%   zDeiTIntermediate.__init__-  s]    YYv1163K3KL
f''-'-f.?.?'@D$'-'8'8D$r8   r   r!   c                 J    | j                  |      }| j                  |      }|S r   )r   r   )r5   r   s     r7   rd   zDeiTIntermediate.forward5  s&    

=100?r8   	rf   rg   rh   r   r%   r'   rk   rd   rn   ro   s   @r7   r   r   ,  s*    9z 9U\\ ell r8   r   c                   t     e Zd Zdef fdZdej                  dej                  dej                  fdZ xZS )
DeiTOutputr   c                     t         |           t        j                  |j                  |j
                        | _        t        j                  |j                        | _	        y r   )
r$   r%   r   r   r   r)   r   r1   r2   r3   r   s     r7   r%   zDeiTOutput.__init__=  sB    YYv779K9KL
zz&"<"<=r8   r   r   r!   c                 T    | j                  |      }| j                  |      }||z   }|S r   r   r   s      r7   rd   zDeiTOutput.forwardB  s.    

=1]3%4r8   r   ro   s   @r7   r   r   <  s8    >z >
U\\  RWR^R^ r8   r   c                        e Zd ZdZdef fdZddej                  deej                     dej                  fdZ	 xZ
S )		DeiTLayerz?This corresponds to the Block class in the timm implementation.r   c                 r   t         |           |j                  | _        d| _        t	        |      | _        t        |      | _        t        |      | _	        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y )Nr   eps)r$   r%   chunk_size_feed_forwardseq_len_dimr   r   r   intermediater   r   r   	LayerNormr)   layer_norm_epslayernorm_beforelayernorm_afterr   s     r7   r%   zDeiTLayer.__init__M  s    '-'E'E$&v.,V4 ( "V-?-?VEZEZ [!||F,>,>FDYDYZr8   r   r   r!   c                     | j                  |      }| j                  ||      }||z   }| j                  |      }| j                  |      }| j	                  ||      }|S r   )r   r   r   r   r   )r5   r   r   hidden_states_normattention_outputlayer_outputs         r7   rd   zDeiTLayer.forwardW  sk    !22=A>>*<iH )=8 ++M:((6 {{<?r8   r   )rf   rg   rh   ri   r   r%   r'   rk   r   rd   rn   ro   s   @r7   r   r   J  sB    I[z [U\\ hu||>T `e`l`l r8   r   c                   h     e Zd Zdef fdZddej                  deej                     defdZ	 xZ
S )DeiTEncoderr   c                     t         |           || _        t        j                  t        |j                        D cg c]  }t        |       c}      | _        d| _	        y c c}w re   )
r$   r%   r   r   
ModuleListrangenum_hidden_layersr   layergradient_checkpointing)r5   r   r\   r6   s      r7   r%   zDeiTEncoder.__init__j  sN    ]]uVE]E]?^#_!If$5#_`
&+# $`s   A#r   r   r!   c                 x    t        | j                        D ]  \  }}|||   nd } |||      } t        |      S )N)last_hidden_state)	enumerater   r   )r5   r   r   ilayer_modulelayer_head_masks         r7   rd   zDeiTEncoder.forwardp  sI    (4 	IOA|.7.CilO(HM	I ??r8   r   )rf   rg   rh   r   r%   r'   rk   r   r   rd   rn   ro   s   @r7   r   r   i  s;    ,z ,@U\\ @hu||>T @`o @r8   r   c                       e Zd ZU eed<   dZdZdZdgZdZ	dZ
dZdZeedZdeej$                  ej&                  ej(                  f   dd	fd
Zy	)DeiTPreTrainedModelr   deitrU   Tr   )r   
attentionsr   r!   Nc                    t        |t        j                  t        j                  f      rt        j                  j                  |j                  j                  j                  t        j                        d| j                  j                        j                  |j                  j                        |j                  _        |j                  %|j                  j                  j                          yyt        |t        j                         rJ|j                  j                  j                          |j                  j                  j#                  d       yt        |t$              r|j&                  j                  j                          |j(                  j                  j                          |j*                  j                  j                          |j,                  %|j,                  j                  j                          yyy)zInitialize the weightsr   )meanstdNrX   )rv   r   r   rz   inittrunc_normal_weightdatar   r'   r   r   initializer_ranger   r   zero_r   fill_r   r*   r0   r+   r,   )r5   r   s     r7   _init_weightsz!DeiTPreTrainedModel._init_weights  s_   fryy"))45 "$!6!6""%%emm43DKKDaDa "7 "b$$% MM {{&  &&( '-KK""$MM$$S)/!!'')&&++113%%**002  ,!!&&,,. -	 0r8   )rf   rg   rh   r   __annotations__base_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_supports_flex_attn_supports_attention_backendr   r   _can_record_outputsr   r   r   rz   r   r   r8   r7   r   r   x  sr    $O&*#$N"&"'
/E"))RYY*L$M /RV /r8   r   c                        e Zd Zddedededdf fdZdefdZd Ze	e
	 	 	 	 dd	eej                     d
eej                     deej                     dedee   defd              Z xZS )	DeiTModelr   add_pooling_layerr    r!   Nc                    t         |   |       || _        t        ||      | _        t        |      | _        t        j                  |j                  |j                        | _        |rt        |      nd| _        | j                          y)z
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether to use a mask token for masked image modeling.
        )r    r   N)r$   r%   r   r   r9   r   encoderr   r   r)   r   	layernorm
DeiTPoolerpooler	post_init)r5   r   r  r    r6   s       r7   r%   zDeiTModel.__init__  sm     	 (O"6*f&8&8f>S>ST,=j(4 	r8   c                 .    | j                   j                  S r   )r9   r.   )r5   s    r7   get_input_embeddingszDeiTModel.get_input_embeddings  s    ///r8   c                     |j                         D ]7  \  }}| j                  j                  |   j                  j	                  |       9 y)z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsr  r   r   r   )r5   heads_to_pruner   r   s       r7   _prune_headszDeiTModel._prune_heads  sE    
 +002 	CLE5LLu%//;;EB	Cr8   rU   rV   r   rT   r   c                    |t        d      | j                  || j                  j                        }| j                  j
                  j                  j                  j                  }|j                  |k7  r|j                  |      }| j	                  |||      }| j                  ||      }|j                  }	| j                  |	      }	| j                  | j                  |	      nd}
t        |	|
      S )z
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        Nz You have to specify pixel_values)rV   rT   )r   )r   pooler_output)r}   get_head_maskr   r   r9   r.   r{   r   r   r   r  r   r  r  r   )r5   rU   rV   r   rT   r   expected_dtypeembedding_outputencoder_outputssequence_outputpooled_outputs              r7   rd   zDeiTModel.forward  s     ?@@ &&y$++2O2OP	 99DDKKQQ/'??>:L??/Tl + 
 ,0<<8HT]<+^);;..98<8OO4UY)-'
 	
r8   )TFNNNF)rf   rg   rh   r   rj   r%   r-   r  r  r   r   r   r'   rk   rm   r   r   r   rd   rn   ro   s   @r7   r  r    s    z d [_ lp &0&9 0C  046:,0).(
u||,(
 "%"2"23(
 ELL)	(

 #'(
 +,(
 
$(
  (
r8   r  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )r  r   c                     t         |           t        j                  |j                  |j
                        | _        t        |j                     | _	        y r   )
r$   r%   r   r   r)   pooler_output_sizer   r	   
pooler_act
activationr   s     r7   r%   zDeiTPooler.__init__  s>    YYv1163L3LM
 !2!23r8   r   r!   c                 \    |d d df   }| j                  |      }| j                  |      }|S )Nr   )r   r-  )r5   r   first_token_tensorr'  s       r7   rd   zDeiTPooler.forward  s6     +1a40

#566r8   r   ro   s   @r7   r  r    s*    4z 4
U\\ ell r8   r  ad  
    DeiT Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://huggingface.co/papers/2111.09886).

    <Tip>

    Note that we provide a script to pre-train this model on custom data in our [examples
    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

    </Tip>
    )custom_introc                        e Zd Zdeddf fdZee	 	 	 	 ddeej                     deej                     deej                     ded	ee   defd
              Z xZS )DeiTForMaskedImageModelingr   r!   Nc                 N   t         |   |       t        |dd      | _        t	        j
                  t	        j                  |j                  |j                  dz  |j                  z  d      t	        j                  |j                              | _        | j                          y )NFT)r  r    r#   r   )in_channelsout_channelsrr   )r$   r%   r  r   r   
Sequentialrz   r)   encoder_strideru   PixelShuffledecoderr  r   s     r7   r%   z#DeiTForMaskedImageModeling.__init__  s     fdS	}}II"..#22A58K8KK
 OOF112
 	r8   rU   rV   r   rT   r   c                 h    | j                   |f|||d|}|j                  }|ddddf   }|j                  \  }}	}
t        |	dz        x}}|j	                  ddd      j                  ||
||      }| j                  |      }d}|| j                  j                  | j                  j                  z  }|j                  d||      }|j                  | j                  j                  d      j                  | j                  j                  d      j                  d      j                         }t        j                  j                  ||d	      }||z  j!                         |j!                         d
z   z  | j                  j"                  z  }t%        |||j&                  |j(                        S )a;  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, DeiTForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224")
        >>> model = DeiTForMaskedImageModeling.from_pretrained("facebook/deit-base-distilled-patch16-224")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 224, 224]
        ```)rV   r   rT   Nr   r=   r>   r   r#   none)	reductiongh㈵>)lossreconstructionr   r   )r   r   rE   rl   rI   rH   r9  r   rt   r4   repeat_interleaverZ   r   r   rJ   l1_losssumru   r   r   r   )r5   rU   rV   r   rT   r   outputsr&  r]   sequence_lengthru   r:   r;   reconstructed_pixel_valuesmasked_im_lossr@   r`   reconstruction_losss                     r7   rd   z"DeiTForMaskedImageModeling.forward  s   L /8dii/
+%=	/

 /
 "33 *!QrT'24C4I4I1
O\_c122)11!Q:BB:|]cejk &*\\/%B"&;;))T[[-C-CCD-55b$EO11$++2H2H!L""4;;#9#91=1	  #%--"7"7F`lr"7"s1D8==?488:PTCTUX\XcXcXpXppN(5!//))	
 	
r8   r(  )rf   rg   rh   r   r%   r   r   r   r'   rk   rm   rj   r   r   r   rd   rn   ro   s   @r7   r2  r2    s    z d "  046:,0).I
u||,I
 "%"2"23I
 ELL)	I

 #'I
 +,I
 
#I
  I
r8   r2  z
    DeiT Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    c                        e Zd Zdeddf fdZee	 	 	 	 ddeej                     deej                     deej                     de
d	ee   defd
              Z xZS )DeiTForImageClassificationr   r!   Nc                 .   t         |   |       |j                  | _        t        |d      | _        |j                  dkD  r*t        j                  |j                  |j                        nt        j                         | _	        | j                          y NF)r  r   )r$   r%   
num_labelsr  r   r   r   r)   Identity
classifierr  r   s     r7   r%   z#DeiTForImageClassification.__init__m  ss      ++f>	 OUN_N_bcNc"))F$6$68I8IJikititiv 	r8   rU   r   labelsrT   r   c                     | j                   |f||d|}|j                  }| j                  |dddddf         }d}	| | j                  ||| j                  fi |}	t        |	||j                  |j                        S )aZ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, DeiTForImageClassification
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> torch.manual_seed(3)  # doctest: +IGNORE_RESULT
        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> # note: we are loading a DeiTForImageClassificationWithTeacher from the hub here,
        >>> # so the head will be randomly initialized, hence the predictions will be random
        >>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224")
        >>> model = DeiTForImageClassification.from_pretrained("facebook/deit-base-distilled-patch16-224")

        >>> inputs = image_processor(images=image, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> logits = outputs.logits
        >>> # model predicts one of the 1000 ImageNet classes
        >>> predicted_class_idx = logits.argmax(-1).item()
        >>> print("Predicted class:", model.config.id2label[predicted_class_idx])
        Predicted class: Polaroid camera, Polaroid Land camera
        ```r   rT   Nr   )r=  logitsr   r   )r   r   rM  loss_functionr   r   r   r   )
r5   rU   r   rN  rT   r   rB  r&  rQ  r=  s
             r7   rd   z"DeiTForImageClassification.forwardy  s    T /8dii/
%=/
 	/
 "33Aq!9: %4%%ffdkkLVLD$!//))	
 	
r8   r(  )rf   rg   rh   r   r%   r   r   r   r'   rk   rj   r   r   r   rd   rn   ro   s   @r7   rH  rH  f  s    
z 
d 
  04,0)-).=
u||,=
 ELL)=
 &	=

 #'=
 +,=
 
=
  =
r8   rH  zC
    Output type of [`DeiTForImageClassificationWithTeacher`].
    c                       e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eej                     ed<   dZeeej                        ed<   dZeeej                        ed<   y)+DeiTForImageClassificationWithTeacherOutputaj  
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores as the average of the cls_logits and distillation logits.
    cls_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores of the classification head (i.e. the linear layer on top of the final hidden state of the
        class token).
    distillation_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores of the distillation head (i.e. the linear layer on top of the final hidden state of the
        distillation token).
    NrQ  
cls_logitsdistillation_logitsr   r   )rf   rg   rh   ri   rQ  r   r'   FloatTensorr  rU  rV  r   r   r   r  r8   r7   rT  rT    s}    	 +/FHU&&'..2J**+27;%"3"34;8<M8E%"3"345<59Ju00129r8   rT  a  
    DeiT Model transformer with image classification heads on top (a linear layer on top of the final hidden state of
    the [CLS] token and a linear layer on top of the final hidden state of the distillation token) e.g. for ImageNet.

    .. warning::

           This model supports inference-only. Fine-tuning with distillation (i.e. with a teacher) is not yet
           supported.
    c                        e Zd Zdeddf fdZee	 	 	 d
deej                     deej                     de
dee   def
d	              Z xZS )%DeiTForImageClassificationWithTeacherr   r!   Nc                    t         |   |       |j                  | _        t        |d      | _        |j                  dkD  r*t        j                  |j                  |j                        nt        j                         | _	        |j                  dkD  r*t        j                  |j                  |j                        nt        j                         | _
        | j                          y rJ  )r$   r%   rK  r  r   r   r   r)   rL  cls_classifierdistillation_classifierr  r   s     r7   r%   z.DeiTForImageClassificationWithTeacher.__init__  s      ++f>	 AG@Q@QTU@UBIIf((&*;*;<[][f[f[h 	 AG@Q@QTU@UBIIf((&*;*;<[][f[f[h 	$
 	r8   rU   r   rT   r   c                 
    | j                   |f||d|}|j                  }| j                  |d d dd d f         }| j                  |d d dd d f         }||z   dz  }	t	        |	|||j
                  |j                        S )NrP  r   r   r#   )rQ  rU  rV  r   r   )r   r   r[  r\  rT  r   r   )
r5   rU   r   rT   r   rB  r&  rU  rV  rQ  s
             r7   rd   z-DeiTForImageClassificationWithTeacher.forward  s     /8dii/
%=/
 	/
 "33((Aq)AB
"::?1aQR7;ST 22a7:! 3!//))
 	
r8   )NNF)rf   rg   rh   r   r%   r   r   r   r'   rk   rj   r   r   rT  rd   rn   ro   s   @r7   rY  rY    s    z d "  04,0).	
u||,
 ELL)
 #'	

 +,
 
5
  
r8   rY  )rH  rY  r2  r  r   )r   )@ri   collections.abcrw   dataclassesr   typingr   r   r   r'   torch.utils.checkpointr   activationsr	   modeling_layersr
   modeling_outputsr   r   r   r   modeling_utilsr   r   processing_utilsr   pytorch_utilsr   r   utilsr   r   r   r   r   utils.genericr   r   configuration_deitr   
get_loggerrf   loggerModuler   r-   rk   floatr   r   r   r   r   r   r   r   r   r  r  r2  rH  rT  rY  __all__r  r8   r7   <module>rp     sP     ! , ,    ! 9  G & Q X X A * 
		H	%VRYY Vr")) P %II%<<% 
% <<	%
 U\\*% % %>1.		 1.jRYY $BII @ryy  
 
* >@")) @ !// !/ !/H I
# I
 I
Z  	]
!4 ]
]
@ L
!4 L
L
^ 
:+ : :& 
0
,? 0

0
fr8   