
    h                    2   d Z ddlmZ ddlmZmZmZmZ ddlZddl	m
c mZ ddlZddlm
Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZmZm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z0m1Z1  e'       rddl2m3Z3 ddl4m5Z5  e(jl                  e7      Z8e e%d       G d de                    Z9e e%d       G d de                    Z:	 	 	 	 dBdZ;g fd Z< G d! d"e
jz                        Z> G d# d$e
j~                        Z@ G d% d&e
j                        ZB G d' d(ej                  j                        ZCd) ZDdCd*ZE G d+ d,e
j                        ZF	 dDd-e
j                  d.ej                  d/ej                  d0ej                  d1eej                     d2eHd3eHfd4ZI G d5 d6e
j                        ZJ G d7 d8e      ZK G d9 d:e      ZLe% G d; d<e              ZMe% G d= d>eM             ZN G d? d@eMe      ZOg dAZPy)EzPyTorch Idefics model.    )	dataclass)AnyCallableOptionalUnionN)nn   )ACT2FN)CacheDynamicCache)GenerationMixin)AttentionMaskConverter)FlashAttentionKwargs)GradientCheckpointingLayer)ModelOutput)ALL_ATTENTION_FUNCTIONSPretrainedConfigPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupleis_torch_flex_attn_availablelogging)deprecate_kwarg   )IdeficsConfig)IdeficsPerceiverResampler)IdeficsVisionEmbeddingsIdeficsVisionTransformer)	BlockMask)make_flex_block_causal_maskz{
    Base class for Idefics model's outputs that may also contain a past key/values (to speed up sequential decoding).
    )custom_introc                       e Zd ZU dZdZeej                     ed<   dZ	ee
e
ej                           ed<   dZee
ej                        ed<   dZee
ej                        ed<   dZee
ej                        ed<   y)IdeficsBaseModelOutputWithPastal  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the model.

        If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
        hidden_size)` is output.
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
        `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and optionally if
        `config.is_encoder_decoder=True` 2 additional tensors of shape `(batch_size, num_heads,
        encoder_sequence_length, embed_size_per_head)`.

        Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if
        `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values`
        input) to speed up sequential decoding.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.

        image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    Nlast_hidden_statepast_key_valueshidden_states
attentionsimage_hidden_states)__name__
__module____qualname____doc__r&   r   torchFloatTensor__annotations__r'   tupler(   r)   r*        j/var/www/html/eduruby.in/venv/lib/python3.12/site-packages/transformers/models/idefics/modeling_idefics.pyr%   r%   7   s    , 6:x 1 129AEOXeE%*;*;$<=>E8<M8E%"3"345<59Ju00129>B%(9(9":;Br4   r%   zS
    Base class for Idefics causal language model (or autoregressive) outputs.
    c                      e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                        ed<   dZeeej                        ed<   dZeeej                        ed<   dZeeej                        ed<   y)	IdeficsCausalLMOutputWithPasta  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
        `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.

        image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    Nlosslogitsr'   r(   r)   r*   )r+   r,   r-   r.   r8   r   r/   r0   r1   r9   r'   listr(   r2   r)   r*   r3   r4   r5   r7   r7   [   s    $ )-D(5$$
%,*.FHU&&'.9=OXd5#4#456=8<M8E%"3"345<59Ju00129>B%(9(9":;Br4   r7   c                    t        j                  | j                  d         j                  dd      j	                  d|      j                  d      j                  | j                        }| j                  d|      } |j                  d      |d<   |j                  d      |d<   |j                  d      |d<   |j                  d      |d<   d|v r|d   }|j                  d|      |d<   ||j                  d|      |d	<   |d   |d   j                  d|      |d<   |d   |d   j                  d|      |d<   | |fS |d   |d   j                  d|      |d<   | |fS |d   |d   j                  d|      |d<   | |fS )
Nr   r   pixel_valuesimage_encoder_embeddingsperceiver_embeddingsimage_attention_masktoken_type_idsattention_mask)	r/   arangeshapeviewrepeattodeviceindex_selectget)	input_idsexpand_sizeis_encoder_decoderrB   encoder_outputsmodel_kwargsexpanded_return_idxrA   s           r5   expand_inputs_for_generationrQ   |   s    	Y__Q'(--b!4;;A{KPPQSTWWXaXhXhi  &&q*=>I#/#3#3N#CL /;/?/?@Z/[L+,+7+;+;<R+SL'(+7+;+;<R+SL'(<'%&67)7)D)DQH[)\%&!)7)D)DQH[)\%&*+7/;<R/S/`/`"0
+, N#/'3N'C'P'PQRTg'h^$ l"" 
0	1	=3?@Z3[3h3h"4
/0 l"" 
,	-	9/;<R/S/`/`"0
+, l""r4   c                 2   t         j                  t         j                  t         j                  d}|D cg c]  }||   	 }}| j	                         D ];  |r&t        fd|D              rj                  d       +j                  d       = | S c c}w )N)	LayerNormLinear	Embeddingc              3   6   K   | ]  }t        |        y wN)
isinstance).0tmodules     r5   	<genexpr>zfreeze_model.<locals>.<genexpr>   s     $]qZ%:$]s   TF)r   rS   rT   rU   modulesanyrequires_grad_)modelmodule_exceptionsmappingmmodule_exceptions_mappedr[   s        @r5   freeze_modelre      s    \\))\\G
 5FFq
FF--/ )$]D\$]!]!!$'!!%(	)
 L  Gs   Bc                   N     e Zd ZdZ	 	 	 	 ddee   ddf fdZd ZdefdZ	 xZ
S )	IdeficsDecoupledEmbeddinga  
    Implements a decoupling of parameters to allow freezing (or not) a subset of the embeddings. In practise, the
    regular `weight` can be trained or frozen (i.e. `partially_freeze=True`), and if `num_additional_embeddings` > 0,
    then it will create `num_additional_embeddings` additional parameters that are always trained. If
    `num_additional_embeddings=0`, then the module defaults back to the regular behavior of `nn.Embedding`.
    Npartially_freezereturnc           	      B   |||kD  rt        d| d|       t        	|   d|||||d| || _        || _        || _        || _        |r| j                  j                  d       | j
                  dkD  r)t        j                  | j
                  |||      | _        yy)	a)  
        Args:
            num_embeddings (`int`):
                Size of the dictionary of embeddings
            num_additional_embeddings (`int`):
                Number of additional embeddings. Only useful when you `partially_freeze=True`.
            embedding_dim (`int`):
                The size of each embedding vector
            partially_freeze: (`bool`, *optional*, defaults to `False`):
                If `True`, the regular `weight` will be frozen. `additional_weight` is never frozen.
            padding_idx (`int`, *optional*):
                The padding index (needs to be less than num_embeddings)

        Note: there are a lot of other parameters to initialize a standard `nn.Embedding` such as `padding_idx`,
        `max_norm` or `norm_type`. We are not supporting these.
        Nz/padding_idx must be within num_embeddings. Got z and )num_embeddingsembedding_dimrH   dtypepadding_idxFr   )rk   rl   rH   rm   r3   )
ValueErrorsuper__init__rk   rn   num_additional_embeddingsrh   weightr_   r   rU   additional_embedding)
selfrk   rr   rl   rh   rH   rm   rn   kwargs	__class__s
            r5   rq   z"IdeficsDecoupledEmbedding.__init__   s    6 "{^'CN{m[`ao`pqrr 	
)'#	
 	
 -&)B& 0KK&&u-))A-(*#==+	)D% .r4   c                 b   | j                   dk(  r t        j                  || j                        S |j	                         }t        j                  || j                  k\        }||   }| j                  || j                  z
        }d||<   t        j                  || j                        }|||<   |S )a  
        we have 2 embeddings, with different indices - one pretrained self.weight and another
        self.additional_embedding.weight that is being trained.

        in order to make a lookup of the input ids, we:
        1. find out the indices of the entries belonging to the 2nd embedding
        2. extract those values while subtracting the size of the first embedding (num_embeddings), since the 2nd
           embedding starts from 0 and not num_embeddings
        3. perform the 2nd embedding lookup
        4. now we handle the 1st embedding, we overwrite indices belonging to the 2nd embedding with a padding index
        5. perform the 1st embedding lookup
        6. now we overwrite the values in the 1st embedding lookup with the values of the 2nd embedding lookup

        note: for the 1st embedding lookup we could have looked up only the low indices and not do the padding, but
        then we have to create a new tensor and populate it with 2 tensors that are spread out across various indices -
        i.e. not a simple concat - I haven't benchmarked the complex case if it's any faster, given that seqlens are
        usually relatively short it's probably not faster or if faster not by much - but might be a good idea to
        measure.

        r   )	rr   F	embeddingrs   cloner/   whererk   rt   )ru   rK   additional_vocab_indicesinput_ids_additional_vocabadditional_embeddingsfull_vectors         r5   forwardz!IdeficsDecoupledEmbedding.forward   s    * ))Q.;;y$++66 OO%	#(;;yD<O<O/O#P %./G%H" $ 9 9:TW[WjWj:j k /0	*+kk)T[[9 1F,-r4   c                 n    d| j                    d| j                   d| j                   d| j                   S )Nznum_embeddings=z, num_additional_embeddings=z, embedding_dim=, partially_freeze=)rk   rr   rl   rh   ru   s    r5   
extra_reprz$IdeficsDecoupledEmbedding.extra_repr  sq     !4!4 55QRVRpRpQq  rB  CG  CU  CU  BV  Vi  jn  j  j  i@  A  	Ar4   )FNNN)r+   r,   r-   r.   r   boolrq   r   strr   __classcell__rw   s   @r5   rg   rg      sH     ,13
 #4.3 
3j%NAC Ar4   rg   c                        e Zd ZdZ	 	 	 	 	 ddedededededdf fd	Zd
ej                  dej                  fdZ	de
fdZ xZS )IdeficsDecoupledLineara  
    Implements a decoupling of parameters to allow freezing (or not) a subset of the parameters. In practise, the
    regular `weight` can be trained or frozen (i.e. `partially_freeze=True`), and if `out_additional_features` > 0,
    then it will create `out_additional_features * in_features` additional parameters that are always trained. If
    `out_additional_features=0`, then the module defaults back to the regular behavior of `nn.Linear`.
    Nin_featuresout_featuresout_additional_featuresbiasrh   ri   c                 "   t         |   |||||       || _        || _        || _        || _        |r8| j                  j                  d       |r| j                  j                  d       |dkD  r t        j                  |||||      | _        yy)aG  
        out_additional_features: int. Number of additional trainable dimensions. Only makes sense when
        `partially_freeze=True`. partially_freeze: bool. If True, the regular `weight` will be frozen and extra
        parameters (if any) will be trainable. If False, default to the regular behavior of nn.Linear.
        Fr   )r   r   r   rH   rm   N)rp   rq   r   rh   r   r   rs   r_   r   r   rT   additional_fc)	ru   r   r   r   r   rh   rH   rm   rw   s	           r5   rq   zIdeficsDecoupledLinear.__init__*  s     	lD&%H'>$ 0&(KK&&u-		((/"Q&!#'4"D 'r4   inputc                     t        j                  || j                  | j                        }| j                  dkD  r)| j                  |      }t        j                  ||fd      }|S )Nr   r<   )ry   linearrs   r   r   r   r/   cat)ru   r   outputadditional_featuress       r5   r   zIdeficsDecoupledLinear.forwardN  sV    %dii8''!+"&"4"4U";YY(;<bAFr4   c           
          d| j                    d| j                   d| j                   d| j                  du d| j                   
S )z=Overwriting `nn.Linear.extra_repr` to include new parameters.zin_features=z, out_features=z, out_additional_features=z, bias=Nr   r   r   r   r   rh   r   s    r5   r   z!IdeficsDecoupledLinear.extra_reprW  s    d../t?P?P>QQklp  mI  mI  lJ  JQ  RV  R[  R[  cg  Rg  Qh  h{  |@  |Q  |Q  {R  S  	Sr4   )r   TTNN)r+   r,   r-   r.   intr   rq   r/   Tensorr   r   r   r   r   s   @r5   r   r   !  s     ()!%"" " "%	"
 " " 
"HU\\ ell SC Sr4   r   c                   ,     e Zd Zd fd	Zd Zd Z xZS )IdeficsRMSNormc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z=
        IdeficsRMSNorm is equivalent to T5LayerNorm
        N)rp   rq   r   	Parameterr/   onesrs   variance_epsilon)ru   hidden_sizeepsrw   s      r5   rq   zIdeficsRMSNorm.__init__^  s1     	ll5::k#:; #r4   c                    |j                  t        j                        j                  d      j	                  dd      }|t        j
                  || j                  z         z  }| j                  j                  t        j                  t        j                  fv r%|j                  | j                  j                        }| j                  |z  S )N   r<   T)keepdim)rG   r/   float32powmeanrsqrtr   rs   rm   float16bfloat16)ru   r(   variances      r5   r   zIdeficsRMSNorm.forwardf  s     ##EMM266q9>>r4>P%Ht?T?T4T(UU ;; ??),,T[[->->?M{{]**r4   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)r2   rs   rD   r   r   s    r5   r   zIdeficsRMSNorm.extra_reprp  s*    ))*+6$2G2G1HIIr4   )gư>)r+   r,   r-   rq   r   r   r   r   s   @r5   r   r   ]  s    $+Jr4   r   c                   .     e Zd Zd fd	Zd ZddZ xZS )IdeficsEmbeddingc                    t         |           || _        || _        || _        d| j                  t        j                  d| j                  dt
        j                        j                  |t
        j                        | j                  z  z  z  }| j                  d|d       | j                  || j                  j                  t        j                         	       y )
N      ?r   r   rm   rH   rm   inv_freqF
persistentseq_lenrH   rm   )rp   rq   dimmax_position_embeddingsbaser/   rC   int64rG   floatregister_buffer_set_cos_sin_cacher   rH   get_default_dtype)ru   r   r   r   rH   r   rw   s         r5   rq   zIdeficsEmbedding.__init__v  s    '>$	IIQ!5;;?BB&X]XcXcBdgkgogooq
 	ZeD 	+DMM4H4HPUPgPgPi 	  	
r4   c                    || _         t        j                  | j                   |t        j                        j	                  | j
                        }t        j                  d|| j
                        }t        j                  ||fd      }| j                  d|j                         j                  |      d       | j                  d|j                         j                  |      d       y )	Nr   zi,j->ijr<   r   
cos_cachedFr   
sin_cached)max_seq_len_cachedr/   rC   r   type_asr   einsumr   r   cosrG   sin)ru   r   rH   rm   rZ   freqsembs          r5   r   z#IdeficsEmbedding._set_cos_sin_cache  s    ")LL00u{{S[[\`\i\ijY4==9iiB/\3779<<+>5Q\3779<<+>5Qr4   c                    || j                   kD  r(| j                  ||j                  |j                         | j                  d | j                  |j                        | j                  d | j                  |j                        fS )Nr   r   )r   r   rH   rm   r   rG   r   )ru   xr   s      r5   r   zIdeficsEmbedding.forward  sy    T,,,##GAHHAGG#T OOHW%((qww(7OOHW%((qww(7
 	
r4   )i   i'  NrW   )r+   r,   r-   rq   r   r   r   r   s   @r5   r   r   u  s    
"R
r4   r   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr<   r   r   )rD   r/   r   )r   x1x2s      r5   rotate_halfr     sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r4   c                     ||   j                  |      }||   j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )an  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`):
            The position indices of the tokens corresponding to the query and key tensors. For example, this can be
            used to pass offsetted position ids when working with a KV-cache.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkr   r   position_idsunsqueeze_dimq_embedk_embeds           r5   apply_rotary_pos_embr     sg    * l

%
%m
4C
l

%
%m
4C3w;q>C/0G3w;q>C/0GGr4   c                   2     e Zd Zdededef fdZd Z xZS )
IdeficsMLPr   intermediate_size
hidden_actc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        t        |   | _        y )NFr   )	rp   rq   r   rT   	gate_proj	down_projup_projr
   act_fn)ru   r   r   r   rw   s       r5   rq   zIdeficsMLP.__init__  s[     	;0AN#4kNyy.?eLZ(r4   c                     | j                  | j                  | j                  |            | j                  |      z        S rW   )r   r   r   r   )ru   r   s     r5   r   zIdeficsMLP.forward  s0    ~~dkk$..*;<t||ANOOr4   )r+   r,   r-   r   r   rq   r   r   r   s   @r5   r   r     s*    
)
) 
) 	
)Pr4   r   r[   querykeyvaluerB   scalingdropoutc                    t        j                  ||j                  dd            |z  }|||z   }t        j                  j                  |dt         j                        j                  |j                        }t        j                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )Nr<   )r   rm   ptrainingr   r   )r/   matmul	transposer   
functionalsoftmaxr   rG   rm   r   r   
contiguous)
r[   r   r   r   rB   r   r   rv   attn_weightsattn_outputs
             r5   eager_attention_forwardr     s     <<s}}R'<=GL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|U3K''1-88:K$$r4   c                       e Zd ZdZ	 	 	 	 	 ddededededededee   f fd	Z	d
e
j                  dedefdZ eddd      	 	 	 	 	 	 	 dde
j                  dee
j                     dee
j                     dee
j                     deee
j                        dededee
j                     dee
j                  ee
j                     eee
j                        f   fd       Z xZS )IdeficsAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr   	num_headsr   is_cross_attentionconfigqk_layer_norms	layer_idxc                    t         	|           || _        || _        || _        ||z  | _        || _        d| _        | j
                  dz  | _        || _	        |-t        j                  d| j                  j                   d       | j
                  |z  | j                  k7  rt        d| j                   d| d      || _        t!        t"        j$                  d      st        d	      | j                  rt!        |j&                  d
      s| j                  n|j&                  j(                  }t#        j*                  | j                  || j
                  z  d      | _        t#        j*                  ||| j
                  z  d      | _        t#        j*                  ||| j
                  z  d      | _        nt#        j*                  | j                  || j
                  z  d      | _        t#        j*                  | j                  || j
                  z  d      | _        t#        j*                  | j                  || j
                  z  d      | _        t#        j*                  || j
                  z  |d      | _        t5        | j
                        | _        || _        | j8                  rMt;        | j
                  |j<                        | _        t;        | j
                  |j<                        | _         y y )NTg      zInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.z?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).scaled_dot_product_attentionz)this model requires pytorch 2.0 or higher	embed_dimFr   r   )!rp   rq   r   r   r   head_dimr   	is_causalr   r   loggerwarning_oncerw   r+   ro   r   hasattrr   r   vision_configr  rT   q_projk_projv_projo_projr   
rotary_embr   r   rms_norm_epsq_layer_normk_layer_norm)
ru   r   r   r   r   r   r   r   kv_input_dimrw   s
            r5   rq   zIdeficsAttention.__init__  s    	&"#y0}}d*" !8!8 9 :, , MMI%$*:*::QRVRbRbQc$YKr3 
 #5r}}&DEHII""(/0D0Dk(R  X^XlXlXvXv  ))  DMM)DK
 ))L)dmm2KRWXDK))DMM)DK ))  DMM)DK
 ))  DMM)DK
 ))  DMM)DK
 ii%

 +4==9, .t}}&BUBU VD .t}}&BUBU VD r4   tensorr   bszc                     |j                  ||| j                  | j                        j                  dd      j	                         S )Nr   r   )rE   r   r  r   r   )ru   r  r   r  s       r5   _shapezIdeficsAttention._shape>  s7    {{3GQQRSUVWbbddr4   past_key_valuer'   4.58new_nameversionr(   key_value_statesrB   r   output_attentions	use_cachecache_positionri   c	                 >   | j                   xs |d u}
|j                         \  }}}| j                  |      j                  ||| j                  | j
                        j                  dd      }|
s| j                  |      j                  ||| j                  | j
                        j                  dd      }| j                  |      j                  ||| j                  | j
                        j                  dd      }n|j                         \  }}}| j                  |      j                  ||| j                  | j
                        j                  dd      }| j                  |      j                  ||| j                  | j
                        j                  dd      }|j                  d   }|||d   z  }|
s2| j                  |t        ||            \  }}t        |||||      \  }}|%d|i}|j                  ||| j                  |      \  }}| j                  r"| j!                  |      }| j#                  |      }t$        }| j&                  j(                  dk7  rN| j&                  j(                  dk(  r|rt*        j-                  d	       nt.        | j&                  j(                     } || ||||f| j0                  sd
n| j2                  | j4                  d|	\  }}|j7                  ||d      j9                         }| j;                  |      }|rd }||fS )Nr   r   r   r   )r   r  eagersdpaz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.        )r   r   r<   )r   sizer
  rE   r   r  r   r  r  rD   r  maxr   updater   r   r  r  r   r   _attn_implementationr  r  r   r   r   r   reshaper   r  )ru   r(   r  rB   r   r'   r  r  r  rv   r   r  q_len_query_states
key_statesvalue_stateskv_len
kv_seq_lenr   r   cache_kwargsattention_interfacer   r   s                            r5   r   zIdeficsAttention.forwardA  s    "44T8HPT8T%**,UA{{=166sE4>>SWS`S`akklmopq!]388eT^^UYUbUbcmmnoqrsJ;;}5::3t~~W[WdWdeoopqstuL+002LAvq%56;;CY]YfYfgqqrsuvwJ,-223PTP]P]^hhijlmn   %%b)
&.++J!|SU=STHC';L*VY[^`l'm$L* &,n=L'6'='=j,X\XfXfht'u$J,,\:L**:6J(?;;++w6{{//69>O##L
 '>dkk>^>^&_#$7	%
  $}}C$,,LL	%
 	%
!\ "))#ub9DDFkk+.LL((r4   )r#  FNFNNNNNFFN)r+   r,   r-   r.   r   r   r   r   r   rq   r/   r   r  r   
LongTensorr2   r   r   r   s   @r5   r   r     s   G #(#'$#'OWOW OW 	OW
 !OW !OW OW C=OWbeU\\ eC ec e %0A6R 4815379="'59J)||J) #5<<0J) !.	J)
 u//0J) "%"56J)  J) J) !!1!12J) 
u||Xell3XeELL>Q5RR	SJ) SJ)r4   r   c                   f    e Zd Zddedee   f fdZ eddd      	 	 	 	 	 	 ddej                  d	eej                     d
eej                     deeej                        dee   dee   deej                     deej                  eeej                  ej                  f      f   fd       Z xZS )IdeficsDecoderLayerr   r   c                    t         |           |j                  | _        t        | j                  |j                  |j
                  ||      | _        t        | j                  |j                  |j                        | _
        t        |j                  |j                        | _        t        |j                  |j                        | _        |j
                  | _        y )N)r   r   r   r   r   r   r   r   r  )rp   rq   r   r   num_attention_headsr   	self_attnr   r   r   mlpr   r  input_layernormpost_attention_layernormru   r   r   rw   s      r5   rq   zIdeficsDecoderLayer.__init__  s    !--)((00NN
 (($66((

  .f.@.@fFYFYZ(6v7I7IvObOb(c%~~r4   r  r'   r  r  r(   rB   r   r  r  r  ri   c                    |}	| j                  |      } | j                  d|||||||d|\  }}
t        j                  j	                  || j                  | j
                        }|	|z   }|}	| j                  |      }| j                  |      }t        j                  j	                  || j                  | j
                        }|	|z   }|f}|r||
fz  }|S )a^  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_values (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
        )r(   rB   r   r'   r  r  r  r   r3   )r;  r9  r   r   r   r   r<  r:  )ru   r(   rB   r   r'   r  r  r  rv   residualself_attn_weightsoutputss               r5   r   zIdeficsDecoderLayer.forward  s    4 !,,]; ,:4>> 	,
')%+/)	,
 	,
(( --mt||VZVcVc-d =0 !55mD/--mt||VZVcVc-d =0 ")++Gr4   rW   )NNNFFN)r+   r,   r-   r   r   r   rq   r   r/   r   r3  r2   r   r0   r   r   r   s   @r5   r5  r5    s    &} &# && %0A6R 26379=,1$)597||7 !.7 u//0	7
 "%"567 $D>7 D>7 !!1!127 
u  (51B1BEDUDU1U+V"WW	X7 S7r4   r5  c                       e Zd Zddedee   f fdZ eddd      	 	 	 	 	 	 	 ddej                  d	eej                     d
eej                     deej                     deej                     dee
   dee
   deeej                        deej                  eeej                  ej                  f      f   fd       Z xZS )IdeficsGatedCrossAttentionLayerr   r   c           	      	   t         |           |j                  | _        t        | j                  |j                  d|j
                  ||j                  |      | _        t        | j                  |j                  |j                        | _        t        |j                  |j                        | _        t        |j                  |j                        | _        |j
                  | _        t#        j$                         | _        t#        j$                         | _        |j*                  dk(  r|j,                  dk(  rtt#        j.                  t1        j2                  dd| j                              | _        t#        j.                  t1        j2                  dd| j                              | _        n|j,                  dk(  r\t#        j.                  t1        j2                  d            | _        t#        j.                  t1        j2                  d            | _        nt9        d	|j,                   d
      |j*                  dk(  r|j,                  dk(  rtt#        j.                  t1        j:                  dd| j                              | _        t#        j.                  t1        j:                  dd| j                              | _        n|j,                  dk(  r\t#        j.                  t1        j:                  d            | _        t#        j.                  t1        j:                  d            | _        n|t9        d	|j,                   d
      |j*                  dv r;|j,                  dk(  rt#        j.                  t1        j<                  d|j>                  dd| j                  f            | _        t#        j.                  t1        j<                  d|j>                  dd| j                  f            | _        n|j,                  dk(  rut#        j.                  t1        j<                  d|j>                  d            | _        t#        j.                  t1        j<                  d|j>                  d            | _        n2t9        d	|j,                   d
      tA        d|j*                   d      tC        | d      rtC        | d      st9        d      y )NT)r   r   r   r   r   r   r   r7  r  zerosvectorr   r   z Unknown value for `alpha_type` ()r   >   normalrandomgaussianr#  )r   stdr$  zAlpha initialization scheme z not yet implemented!alpha_cross_attnalpha_densez+Alpha parameters not initialized correctly!)"rp   rq   r   r   r8  r   r   
cross_attnr   r   r   r:  r   r  r;  r<  r   r   Tanhact_cross_attn	act_densealpha_initializer
alpha_typer   r/   rE  rL  rM  ro   r   rH  alphas_initializer_rangeNotImplementedErrorr  r=  s      r5   rq   z(IdeficsGatedCrossAttentionLayer.__init__  s   !--*((00#NN!00
 (($66((

  .f.@.@fFYFYZ(6v7I7IvObOb(c%nn ggi##w.  H,(*U[[AtGWGW5X(Y%#%<<Aq$BRBR0S#T ""g-(*U[[^(D%#%<<A#?  #CFDUDUCVVW!XYY%%/  H,(*UZZ1dFVFV5W(X%#%<<

1aAQAQ0R#S ""g-(*UZZ](C%#%<<

1#>  #CFDUDUCVVW!XYY%%)II  H,(*LLcv/N/NVWYZ\`\l\lUmn)% $&<<LLcv/N/NVWYZ\`\l\lUmn$  ""g-(*LLcv/N/NVWY)% $&<<#6KjKjrs0u#v  #CFDUDUCVVW!XYY &(DVE]E]D^^s&tuu01gdM6RJKK 7Sr4   r  r'   r  r  r(   rB   r*   r@   cross_attention_gater  r  ri   c	                    |t        d      |t        d      |t        d      |}
| j                  |      } | j                  d	||||d|	\  }}t        j
                  j                  || j                  | j                        }|j                  |dk(  dddddf   d      }|
| j                  | j                        |z  z   }|}
| j                  |      }| j                  |      }t        j
                  j                  || j                  | j                        }|
| j                  | j                        |z  z   }|f}|r||fz  }|S )
a  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            image_attention_mask (`torch.FloatTensor`, *optional*): image attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            cross_attention_gate (`torch.FloatTensor`, *optional*):
                gate of size `(batch, seq_len)` used to zero-out cross-attention output for tokens attending no images.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_values (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
        Nzt`image_hidden_states` is required for Idefics cross attention module which are visual features to be conditioned on.z`cross_attention_gate` is required for Idefics cross attention module to zero-out the cross-attention hidden_states attending to no images.zMPast key value states are not implemented for Idefics cross attention module.)r(   r  rB   r  r   r   r#  r3   )ro   rU  r;  rN  r   r   r   r   r   masked_fillrP  rL  r<  r:  rQ  rM  )ru   r(   rB   r*   r@   rV  r  r  r'   rv   r?  r@  rA  s                r5   r   z'IdeficsGatedCrossAttentionLayer.forward"  s   < &# 
  ' ^  &%&uvv ,,]; ,;4?? ,
'0//	,

 ,
(( --mt{{UYUbUb-c%113G13LaQRTXj2Y[^_ 4#6#6t7L7L#MP]#]] !55mD/--mt{{UYUbUb-c 4>>$2B2B#Cm#SS ")++Gr4   rW   r2  )r+   r,   r-   r   r   r   rq   r   r/   r   r   r2   r0   r   r   r   s   @r5   rC  rC    s$   @L} @L# @LD %0A6R 266:7;7;,1$)9=H||H !.H &ell3	H
 'u||4H 'u||4H $D>H D>H "%"56H 
u  (51B1BEDUDU1U+V"WW	XH SHr4   rC  c                   >    e Zd ZU eed<   dZdZddgZdZdZ	dZ
dZd Zy)	IdeficsPreTrainedModelr   r`   Tr5  rC  Fc                    | j                   j                  }t        |t        j                  t        j
                  f      rY|j                  j                  j                  d|       |j                  %|j                  j                  j                          y y t        |t        j                        rf|j                  j                  j                  d|       |j                  2|j                  j                  |j                     j                          y y t        |t        j                        rJ|j                  j                  j                  d       |j                  j                  j                          y t        |t              r&|j                  j                  j                  d       y t        |t               r%|j"                  j                  j                          y t        |t$              rV| j                   j&                  dk(  rI|j(                  j                  j                          |j*                  j                  j                          y | j                   j&                  dk(  rK|j(                  j                  j                  d       |j*                  j                  j                  d       y | j                   j&                  dv rw|j(                  j                  j                  d| j                   j,                         |j*                  j                  j                  d| j                   j,                         y y t        |t.              r%|j0                  j                  j                          y y )Nr#  )r   rK  r   rE  r   >   rH  rI  rJ  )r   initializer_rangerX   r   rT   Conv2drs   datanormal_r   zero_rU   rn   rS   fill_r   r   class_embeddingrC  rR  rL  rM  rT  r   latents)ru   r[   rK  s      r5   _init_weightsz$IdeficsPreTrainedModel._init_weightsz  sp    kk++fryy"))45MM&&CS&9{{&  &&( '-MM&&CS&9!!-""6#5#56<<> .-MM$$S)KK""$/MM$$S) 78""''//1 ?@{{,,7'',,224""''--/..&8'',,2237""''--c2..2RR'',,44#4;;CgCg4h""''//Sdkk>b>b/c S  9:NN'') ;r4   N)r+   r,   r-   r   r1   base_model_prefixsupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_can_compile_fullgraph_supports_attention_backendrd  r3   r4   r5   rZ  rZ  n  s<    &*#.0QRN ""&*r4   rZ  c            '           e Zd ZdZdef fdZd!dZg fdZg fdZe	e
	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d"deej                     deej                     d	eej                     d
ee   deej                      deej                      deej                      deej                      deej                     dee   dee   dee   dee   dee   deej                     dee   deeef   f"d              Z	 d#deej                  df   dej                  dej                  d
edef
dZedej                  dededej6                  dej                  defd        Z xZS )$IdeficsModelz
    Transformer decoder consisting of `config.num_hidden_layers` layers. Each layer is a [`IdeficsDecoderLayer`]

    Args:
        config: IdeficsConfig
    r   c           	         t         |   |       || _        |j                  | _        |j
                  | _        t        |j
                  |j                  |j                  |j                  | j                        | _
        |j                  j                  | _        |j                  | _        |j                  | j                  _        t        |j                        | _        |j                   r]|j"                  }t%        ||j                  j&                  |j(                  |j*                  |j,                  |j.                        | _        t3        j4                  t7        |j8                        D cg c]  }t;        ||       c}      | _        |j>                  | _        |j8                  | j>                  z  }t3        j4                  t7        |      D cg c]  }tA        ||       c}      | _!        d| _"        tG        |j                  |jH                        | _%        | jM                          | jO                  |       y c c}w c c}w )N)rk   rr   rl   rh   rn   )r   Fr  )(rp   rq   r   pad_token_idrn   
vocab_sizerg   additional_vocab_sizer   freeze_text_layersembed_tokensr	  
image_sizer'  r    vision_modeluse_resamplerperceiver_configr   r  resampler_depthresampler_n_headsresampler_head_dimresampler_n_latentsperceiver_resamplerr   
ModuleListrangenum_hidden_layersr5  layerscross_layer_intervalrC  gated_cross_attn_layersgradient_checkpointingr   r  norm	post_initfreeze_relevant_params)ru   r   rw  inum_cross_layersrw   s        r5   rq   zIdeficsModel.__init__  s    !.. ++5!,,&,&B&B ,,#66((
 !..99#11282M2M/4V5I5IJ %66'@$$.. 00 22 33 44(D$ mm?DVE]E]?^_! 15_
 %+$?$?!!33t7P7PP')}}KPQaKbca,VqAc(
$ ',#"6#5#56;N;NO	 	##F+ ` ds   1IIc                     || j                   }|j                  r| j                  |j                         |j                  r"t	        | j
                  |j                         y y N)ra   )r   rr  freeze_text_module_exceptionsfreeze_vision_layersre   ru  freeze_vision_module_exceptions)ru   r   s     r5   r  z#IdeficsModel.freeze_relevant_params  sQ    >[[F$$##F$H$HI&&**f>d>de 'r4   c                 X    | j                   | j                  fD ]  }t        ||        y r  )r  r  re   )ru   ra   r[   s      r5   rr  zIdeficsModel.freeze_text_layers  s+    {{DII. 	FF3DE	Fr4   c                 2    t        | j                  |       y r  )re   ru  )ru   ra   s     r5   r  z!IdeficsModel.freeze_vision_layers  s    T&&:KLr4   rK   rB   r   r'   inputs_embedsr=   r>   r?   r@   r  r  output_hidden_statesinterpolate_pos_encodingreturn_dictr  rv   ri   c                 
   ||j                   n|j                   }||n| j                  j                  }||n| j                  j                  }|
|
n| j                  j                  }
||n| j                  j
                  }|du |duz  rt        d      | j                  r%| j                  r|
rt        j                  d       d}
|| j                  |      }t        |t        d      t        f      st        d      |
r|t        | j                        }|j                   \  }}}||j#                         nd}||z   }|2t%        j&                  |||j                   d   z   |j                   	      }|F|D|j)                         j+                  d
      dz
  }|j-                  |dk(  d       |dd| df   }n||j/                  d      }t1        |||fD cg c]  }|du  c}      dk7  rt        d      |~|j3                  | j4                  |      }|j                   dd \  }} |j7                         j8                  ||z  g|j                   dd  }| j;                  ||      j<                  }nJ|H|j?                         \  }}}}|j3                  | j4                  |      }|j9                  ||z  ||      }| j                  j@                  rN|4| jC                        }|j?                  d      |j?                  d      }}n|j?                         \  }}}}|}n0|#j?                  d      |j?                  d      }}nt        d      |j9                  ||z  |      }|	j?                  d      }|	j/                  d
      }	|	jE                  ddd|      }	|	j9                  ||||z        }	|C|j?                         \  }}}||f}|	t%        jF                  ||	      }	| jI                  |	      }	nd}	|	dk(  jK                  d
      j3                  | j4                        jM                  d      j3                  |      } |2t%        jF                  ||ft$        jN                  |j                         }| jQ                  |||||      }|}!|rdnd}"|rdnd}#tS        | jT                        D ]r  \  }$}%|r|"|!fz  }"|$| jV                  z  dk(  r2| jX                  |$| jV                  z     }& |&|!||f|	| ||
dd|}'|'d   }! |%|!f|||||
|d|}(|(d   }!|sj|#|(d   fz  }#t | j[                  |!      }!|r|"|!fz  }"|j9                  ||||      }t]        |!||"|#|      S c c}w )ab  
        image_encoder_embeddings (`torch.FloatTensor`, *optional*):
            The output of the image encoder.
        perceiver_embeddings (`torch.FloatTensor`, *optional*):
            The output of the perceiver resampler.
        image_attention_mask (`torch.LongTensor`, *optional*):
            The attention mask for the image encoder.
        Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.FzBThe `past_key_values` should be either a `Cache` object or `None`.)r   r   r   rH   r<   r   z_Exactly 1 of pixel_values, image_encoder_embeddings or perceiver_embeddings has to be not-None.)rm   rH   )r=   r  zBIf `perceiver_embeddings` are passed, use_resampler should be Truer#  r   r   r3   )r@   rV  r  r  r'   )rB   r   r'   r  r  r  )r&   r'   r(   r)   r*   )/rH   r   r  r  r  use_return_dictro   r  r   r  r  rs  rX   typer   r   rD   get_seq_lengthr/   rC   longcumsummasked_fill_r   sumrG   rm   r   rE   ru  r&   r$  rv  r|  rF   r   invert_attention_maskr^   squeezer   _update_causal_mask	enumerater  r  r  r  r%   ))ru   rK   rB   r   r'   r  r=   r>   r?   r@   r  r  r  r  r  r  rv   rH   
batch_size
seq_lengthr*  past_key_values_lengthseq_length_with_pastr   
num_imagesr*   image_seq_lenimage_hidden_sizetext_seq_lenimage_batch_sizeimage_sequence_lengthimage_hidden_shaperV  r(   all_hidden_statesall_self_attnsidxdecoder_layercross_attn_blockrA  layer_outputss)                                            r5   r   zIdeficsModel.forward  se   : &/%:!!@T@T1B1N-TXT_T_TqTq$8$D $++JjJj 	 "+!6IDKK<Q<Q	%0%<k$++B]B]-t";<YZZ&&4==Yj I  --i8M /DJ+>?abb0*$++>O$1$7$7!
JETE`!?!?!Afg),BB!"\\&(>ATATUVAW(W`m`t`tN %,*>)..077;a?L%%n&91='J;<8L!)33A6LL2JL`#abaT	bcghhq  %'??F?KL%1%7%7%;"J
9<22499*z:QkT`TfTfghgiTjkL #'"3"3)D\ #4 #   &1G_GdGdGfDJ
M3D":"="=DJJW]"="^"5":"::
;RTact"u;;$$#+'+'?'?@S'T$3G3L3LQ3OQeQjQjklQm0K_KdKdKfH
J7H"6!)/B/G/G/JL_LdLdefLg,Mabb166z:P]C]_pq ,0033==bA3::1aMR388\S]`mSmn*9L9Q9Q9S63Q"24I!J#+',zz2DV'T$#'#=#=>R#S #'  $83#>"C"C"C"K!O!OVZV`V`!O!a j jop j quu 

 !"ZZ12%**]MaMaN 11M>?L]
 & #7BD0d"+DKK"8 !	6C#!m%55! T...!3#'#?#?tG`G`@`#a *!"'
 *>)=&7'$(
 
 !(
)	-) /"3#-	 	M *!,M =#3"55C!	6F 		-0  -!11166z:}^op-+++% 3
 	
m cs   U2r!   input_tensorc           	         | j                   j                  dk(  r||dk(  j                         r|S y | j                   j                  dk(  r't        |t        j
                        rt        |      }|S ||j                         nd}||j                  nd}| j                   j                  dk(  r(|s&|s$t        j                  |||| j                        ry |j                  }|j                  d   }	|r|j                         }
n1t        |t        j
                        r|j                  d	   n||	z   dz   }
| j                  ||	|
|||j                  d   
      }| j                   j                  dk(  rQ|O|j                   j"                  dv r7|s5t	        j$                  |      j&                  }t        j(                  ||      }|S )Nflash_attention_2r#  flex_attentionr   Fr"  )r  r  is_trainingr   r<   )sequence_lengthtarget_lengthrm   r  r  )cudaxpunpu)r   r'  r^   rX   r/   r   r"   r  is_compileabler   _ignore_causal_mask_sdpar   rm   rD   get_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positionrH   r  finfomin_unmask_unattended)ru   rB   r  r  r'   r  past_seen_tokensusing_compilable_cacherm   r  r  causal_mask	min_dtypes                r5   r  z IdeficsModel._update_causal_mask  s    ;;++/BB)~/D.I.I.K%%;;++/??.%,,7!<^!L!!
 @O?Z?99;`aCRC^!?!?di ;;++v5>T]n%>>*'7 MM	 ""&,,Q/!+??AM nell; $$R(%7!;  PP+')#))!, Q 
 KK,,6*%%**.DD%
 E*..I0CCKQZ[Kr4   r  r  rm   r  c                    | | j                         dk(  r| }|S t        j                  |      j                  }t        j                  ||f|||j
                        }|dk7  rt        j                  |d      }|t        j                  ||j
                        |j                  dd      kD  z  }|ddddddf   j                  |ddd      }| |j                         }| j                  d   }	|ddddddd|	f   | ddddddf   j                  |j
                        z   }
|
dk(  }
|ddddddd|	f   j                  |
|      |ddddddd|	f<   |S )	aM  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        N   )
fill_valuerm   rH   r   )diagonalr  r<   r   )r   r/   r  r  fullrH   triurC   r(  expandr{   rD   rG   rX  )rB   r  r  rm   r  r  rv   r  r  mask_lengthpadding_masks              r5   r  zBIdeficsModel._prepare_4d_causal_attention_mask_with_cache_position  s   > %.*<*<*>!*C(K* ' E*..I** -0Ye\j\q\qK !##jjqA5<<n>S>STWeWmWmnprsWtttK%dD!Q&67>>z1bRTUK))//1,2226*1aL[L+@ANSTVZ\`bcScDdDgDg&&E    ,q05@Aq,;,AV5W5c5c )6Aq!\k\12 r4   rW   )NNNNNNNNNNNNFNNF)r+   r,   r-   r.   r   rq   r  rr  r  r   r   r   r/   r3  r   r   r0   r   r   r   r   r2   r%   r   r  staticmethodr   rm   r  r   r   s   @r5   rm  rm    so   0,} 0,df 46 F 68 M  151537+/5948@D<@7;$(,0/338&*59!E
E,,-E
 !.E
 u//0	E

 "%E
   1 12E
 u001E
 #+5+<+<"=E
 'u'8'89E
 'u||4E
 D>E
 $D>E
 'tnE
 #+4.E
 d^E
  !!1!12!E
" -.#E
$ 
u44	5%E
  E
\ #(BellK78B llB 	B
 B  BH 444 4 {{	4
 4 4 4r4   rm  c            )       >    e Zd ZddgZd fd	Zd Zee	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddee	j                     dee	j                     dee	j                     dee   d	ee	j                     d
ee	j                     dee	j                     dee	j                     dee	j                     dee	j                     dee   dee   dee   dee   dee   dee	j                     dee   deeef   f$d              Z	 	 	 	 	 	 	 	 	 d fd	Z	 d dedeeef   dedeeef   f fdZ xZS )!IdeficsForVisionText2Textzmodel.embed_tokens.weightzlm_head.weightc                     t         |   |       t        |      | _        t	        |j
                  |j                  |j                  d|j                        | _	        | j                          y )NFr   )rp   rq   rm  r`   r   r   rp  rq  freeze_lm_headlm_headr  )ru   r   ru  rw   s      r5   rq   z"IdeficsForVisionText2Text.__init__2  s[     !&)
-****$*$@$@#22
 	r4   c                    | j                         }| j                         }t        | j                  dd      r`|j                  |_        |j
                  dkD  r@|j                  |j
                  k(  sJ |j                  j                  |j                  _        t        |d      rJt        |d      r=|j                  |_        t        |d      rt        |d      r|j
                  |_        yyyyy)	z
        Overwrite `transformers.modeling_utils.PreTrainedModel.tie_weights` to handle the case of
        IdeficsDecoupledLinear and IdeficsDecoupledEmbedding.
        tie_word_embeddingsTr   r   rk   r   rr   N)get_output_embeddingsget_input_embeddingsgetattrr   rs   rr   r   rt   r   r  rk   r   )ru   output_embeddingsinput_embeddingss      r5   tie_weightsz%IdeficsForVisionText2Text.tie_weightsA  s    
 !6684464;; 5t<'7'>'>$99A=(@@DTDnDnnnn9I9^9^9e9e!//6$n5'BRTd:e-=-L-L*(*CD "=J =M<f<f!9JD ;f5r4   rK   rB   r   r'   r  r=   r>   r?   r@   labelsr  r  r  r  r  r  rv   ri   c                    ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  } | j                  d|||||||||	||||d|d|}|d   }| j                  |      }d}|
* | j                  d||
| j                   j                  d|}t        |||j                  |j                  |j                  |j                        S )aC  
        image_encoder_embeddings (`torch.FloatTensor`, *optional*):
            The output of the image encoder.
        perceiver_embeddings (`torch.FloatTensor`, *optional*):
            The output of the perceiver resampler.
        image_attention_mask (`torch.LongTensor`, *optional*):
            The attention mask for the image encoder.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoProcessor, IdeficsForVisionText2Text

        >>> model = IdeficsForVisionText2Text.from_pretrained("HuggingFaceM4/idefics-9b")
        >>> processor = AutoProcessor.from_pretrained("HuggingFaceM4/idefics-9b")

        >>> dogs_image_url_1 = "https://huggingface.co/datasets/hf-internal-testing/fixtures_nlvr2/raw/main/image1.jpeg"
        >>> dogs_image_url_2 = "https://huggingface.co/datasets/hf-internal-testing/fixtures_nlvr2/raw/main/image2.jpeg"

        >>> prompts = [
        ...     [
        ...         "User:",
        ...         dogs_image_url_1,
        ...         "Describe this image.\nAssistant: An image of two dogs.\n",
        ...         "User:",
        ...         dogs_image_url_2,
        ...         "Describe this image.\nAssistant:",
        ...     ]
        ... ]
        >>> inputs = processor(prompts, return_tensors="pt")
        >>> generate_ids = model.generate(**inputs, max_new_tokens=6)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True)
        ```NT)rK   rB   r   r'   r  r=   r>   r?   r@   r  r  r  r  r  r  r   )r9   r  rp  )r8   r9   r'   r(   r)   r*   r3   )r   r  r  r  r`   r  loss_functionrp  r7   r'   r(   r)   r*   )ru   rK   rB   r   r'   r  r=   r>   r?   r@   r  r  r  r  r  r  r  rv   rA  r(   r9   r8   s                         r5   r   z!IdeficsForVisionText2Text.forwardV  s%   x 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B] $** 
)%+'%%=!5!5/!5%=)
  !
&  
m,%4%%pVFt{{OeOepiopD,#33!//)) ' ; ;
 	
r4   c                    i }|"| j                   j                  r||d<   n||d<   n||d<   |j                  dd      |d<   t        |   |f||||||
|	d||}|	#|!|d   j
                  d   }|	d d | d f   |d	<   |S )
Nr?   r>   r=   r  F)r'   rB   r  r  r   r  r@   rK   r   r@   )r   rv  poprp   prepare_inputs_for_generationrD   )ru   rK   rB   r   r  r'   r  r=   r*   r@   r  rv   images_kwargsmodel_inputsr  rw   s                  r5   r  z7IdeficsForVisionText2Text.prepare_inputs_for_generation  s      *{{((8K45<O89,8M.)4:JJ?Y[`4a01w<
+)')%!5
 
 
  +0E%k288;J3GJ;<3XL/0r4   rA  rO   rM   c                     t        |   |||fi |}d|v rT|d   }|d d dd d f   j                  d      }|j                  dd      r||d<   nt	        j
                  ||gd      |d<   |j                  |d<   |S )Nr@   r<   r   r  Tr   r*   )rp   #_update_model_kwargs_for_generationr   rJ   r/   r   r*   )ru   rA  rO   rM   rv   r@   	last_maskrw   s          r5   r  z=IdeficsForVisionText2Text._update_model_kwargs_for_generation  s     wB
 	
 "\1#/0F#G ,QAX6@@CIT27@347<yyBVXaAbhi7j34 /6.I.I*+r4   rW   )NNNNNNNNNNNNNFNN)	NNNNNNNNNr  )r+   r,   r-   _tied_weights_keysrq   r  r   r   r   r/   r3  r   r   r0   r   r   r   r   r2   r7   r   r  r   dictr   r   r  r   r   s   @r5   r  r  /  s*   57GHg*  151537+/5948@D<@7;-1$(,0/338&*59#b
E,,-b
 !.b
 u//0	b

 "%b
   1 12b
 u001b
 #+5+<+<"=b
 'u'8'89b
 'u||4b
 ))*b
 D>b
 $D>b
 'tnb
 #+4.b
  d^!b
" !!1!12#b
$ +,%b
& 
u33	4'b
  b
N  !+b $)	 38n !	 
c3h r4   r  )r  rm  rZ  )r   FNN)r   )r#  )Qr.   dataclassesr   typingr   r   r   r   r/   torch.nn.functionalr   r   ry   torch.utils.checkpointactivationsr
   cache_utilsr   r   
generationr   modeling_attn_mask_utilsr   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   modeling_utilsr   r   r   processing_utilsr   utilsr   r   r   r   r   utils.deprecationr   configuration_ideficsr   	perceiverr   visionr   r    !torch.nn.attention.flex_attentionr!   integrations.flex_attentionr"   
get_loggerr+   r  r%   r7   rQ   re   rU   rg   rT   r   Moduler   r   r   r   r   r   r   r   r   r5  rC  rZ  rm  r  __all__r3   r4   r5   <module>r     sK  (  ! 1 1      ! . ) > B 9 + X X & p p 0 0 0 E  !;J 
		H	% 
C[ C C< 
CK C C: *#Z +- fA fAR8SRYY 8SxJRYY J0$
uxx $
N(:P P2 %II%<<% 
% <<	%
 U\\*% % %0b)ryy b)LL4 L^L&@ L^ **_ ** **Z O) O OdR 6 Rj Rr4   