
    hl                        d Z ddlmZ ddlmZmZ ddlZddlZddlmZ ddl	m
Z
mZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZmZ ddlmZ ddlmZ  ej@                  e!      Z"e ed       G d de                    Z#e ed       G d de                    Z$ G d dejJ                        Z&e G d de             Z' ed       G d de'             Z( ed        G d! d"e'e             Z)g d#Z*y)$zPyTorch PaliGemmamodel.    )	dataclass)OptionalUnionN)nn   )CacheStaticCache)GenerationMixin)FlashAttentionKwargs)BaseModelOutputWithPast)PreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstringcan_return_tuplelogging   )	AutoModel   )PaliGemmaConfigzN
    Base class for Paligemma outputs, with hidden states and attentions.
    )custom_introc                   :    e Zd ZU dZdZeej                     ed<   y)PaligemmaModelOutputWithPasta  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
        `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    Nimage_hidden_states)	__name__
__module____qualname____doc__r   r   torchFloatTensor__annotations__     n/var/www/html/eduruby.in/venv/lib/python3.12/site-packages/transformers/models/paligemma/modeling_paligemma.pyr   r   ,   s    
 8<%"3"34;r$   r   zU
    Base class for PaliGemma causal language model (or autoregressive) outputs.
    c                      e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeeej                     ef      ed<   dZeeej                        ed<   dZeeej                        ed<   dZeej                     ed<   y)	PaliGemmaCausalLMOutputWithPastaa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
        `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
    Nlosslogitspast_key_valueshidden_states
attentionsr   )r   r   r   r   r(   r   r    r!   r"   r)   r*   r   listr   r+   tupler,   r   r#   r$   r%   r'   r'   B   s      )-D(5$$
%,*.FHU&&'.GKOXeD):):$;U$BCDK8<M8E%"3"345<59Ju001297;%"3"34;r$   r'   c                   *     e Zd Zdef fdZd Z xZS )PaliGemmaMultiModalProjectorconfigc                     t         |           t        j                  |j                  j
                  |j                  j                  d      | _        y )NTbias)super__init__r   Linearvision_confighidden_sizeprojection_dimlinearselfr1   	__class__s     r%   r6   z%PaliGemmaMultiModalProjector.__init__b   s;    ii 4 4 @ @&BVBVBeBelpqr$   c                 (    | j                  |      }|S N)r;   )r=   image_featuresr+   s      r%   forwardz$PaliGemmaMultiModalProjector.forwardf   s    N3r$   )r   r   r   r   r6   rB   __classcell__r>   s   @r%   r0   r0   a   s    r rr$   r0   c                   D    e Zd ZU eed<   dZdZdgZdZdZ	dZ
dZdZdZd Zy)	PaliGemmaPreTrainedModelr1    Tr0   r*   Fc                 \   t        | j                  d| j                  j                         j                        }t	        |t
        j                        rY|j                  j                  j                  d|       |j                  %|j                  j                  j                          y y y )Ninitializer_range        )meanstd)getattrr1   get_text_configrI   
isinstancer   r7   weightdatanormal_r4   zero_)r=   modulerL   s      r%   _init_weightsz&PaliGemmaPreTrainedModel._init_weightsz   s     dkk#68S8S8U8g8ghfbii(MM&&CS&9{{&  &&( ' )r$   N)r   r   r   r   r"   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_can_compile_fullgraph_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendrU   r#   r$   r%   rF   rF   l   sE    &*#78"3"N"&)r$   rF   z{
    The Base Paligemma model which consists of a vision backbone and a language model withou language modeling head.,
    c            #       `    e Zd ZddiZdZdef fdZd Zd Zd Z	d	 Z
	 	 	 	 	 dd
ee   fdZdej                  fdZdej"                  dej                  dej                  fdZee	 	 	 	 	 	 	 	 	 	 	 	 	 d dej"                  dej                  deej*                     deej"                     deeeej                     ef      deej"                     deej"                     deej                     deej"                     dee   dee   dee   dee   dee   deeef   fd              Z xZS )!PaliGemmaModelzlanguage_model.modellanguage_modelFr1   c                    t         |   |       t        j                  |j                        | _        t        |      | _        |j                  j                  | _	        t        j                  |j                        }|| _
        | j                  j                  | j                  j                  nd| _        | j                          y )N)r1   )r5   r6   r   from_configr8   vision_towerr0   multi_modal_projectortext_config
vocab_sizera   r1   pad_token_id	post_init)r=   r1   ra   r>   s      r%   r6   zPaliGemmaModel.__init__   s     %119M9MN%A&%I" ,,77"..f6H6HI,8<8P8P8\DKK44bdr$   c                 6    | j                   j                         S r@   )ra   get_input_embeddingsr=   s    r%   rl   z#PaliGemmaModel.get_input_embeddings   s    ""7799r$   c                 :    | j                   j                  |       y r@   )ra   set_input_embeddingsr=   values     r%   ro   z#PaliGemmaModel.set_input_embeddings   s    007r$   c                     || _         y r@   ra   r=   decoders     r%   set_decoderzPaliGemmaModel.set_decoder   s
    %r$   c                     | j                   S r@   rs   rm   s    r%   get_decoderzPaliGemmaModel.get_decoder   s    """r$   is_trainingc                    | j                   j                  j                  dk(  r	|d|v r|S y ||n| j                  }t	        |t
              }t        j                  | j                        j                  }||}|j                  d d \  }	}
|r|j                         }n4t	        |t        j                        r|j                  d   n
|d   |
z   dz   }||j                         dk(  r|S t        j                  |
|f|| j                  |j                        }|
dk7  r%|rt        j                   |d	      }nd|d d d |
f<   |t        j"                  ||j                  
      |j%                  dd      kD  z  }|d d d d d d f   j'                  |	ddd      }||j)                         }|j                  d   }|rd|t+        d      |d d d d d d d |f   j-                  |d d d d d d f   j/                  |j                        dk(  d      |d d d d d d d |f<   |d d d d d d d |f   |d d d d d d f   j/                  |j                        z   }|dk(  }|d d d d d d d |f   j-                  ||      |d d d d d d d |f<   |S )Nflash_attention_2rJ   r   rc   r   r      
fill_valuedtypedevicediagonalr   z/Token type ids must be provided during training)r1   rg   _attn_implementationtrainingrO   r	   r    finfor   minshapeget_max_cache_shapeTensordimfullr   triuarangereshapeexpandclone
ValueErrormasked_fillto)r=   attention_masktoken_type_idsr*   cache_positioninput_tensorry   using_static_cache	min_dtypeinputs_lead_dimsequence_lengthtarget_lengthcausal_maskmask_lengthpadding_masks                  r%   _update_causal_maskz"PaliGemmaModel._update_causal_mask   s    ;;""77;NN)c^.C%%%0%<k$--'EKK

+//	)L+7+=+=bq+A(+??AM nell; $$R(#A&81<  %.*<*<*>!*C!!jjm,$**]k]r]r
 a#jjqA36A///0u||M.:O:OPSaSiSijlnoSppp!$a"23::?ArSUV%%++-K(..r2K !)$%VWW5@Aq,;,AV5W5c5c"1dD!#34778J8JKqPRS6Aq!\k\12
 'q!Q'<=qRVX\^_O_@`@c@cdodvdv@wwL'1,L1<Q1l{l=R1S1_1_i2K1a+-. r$   pixel_valuesc                     | j                  |      }|j                  }| j                  |      }|| j                  j                  j
                  dz  z  }|S )a  
        Obtains image last hidden states from the vision tower and apply multimodal projection.

        Args:
            pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`)
               The tensors corresponding to the input images.
        Returns:
            image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`).
        g      ?)re   last_hidden_staterf   r1   rg   r9   )r=   r   image_outputsselected_image_featurerA   s        r%   get_image_featuresz!PaliGemmaModel.get_image_features   sW     )),7!.!@!@334JK'4;;+B+B+N+NPS+STr$   	input_idsinputs_embedsrA   c                 P   |m| | j                         t        j                  | j                  j                  t        j
                  |j                              k(  }|j                  d      }n|| j                  j                  k(  }|j                         }|j                  d      j                  |      j                  |j                        }|j                  d   |j                  d   z  }||   j                         |j                         k7  rt        d| d|       |S )z
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        )r   r   rc   r   r   z6Image features and image tokens do not match: tokens: z, features )rl   r    tensorr1   image_token_idlongr   allsum	unsqueeze	expand_asr   r   numelr   )r=   r   r   rA   special_image_maskn_image_tokensn_image_featuress          r%   get_placeholder_maskz#PaliGemmaModel.get_placeholder_mask   s    !.2M$2K2K2MT[[77uzzR_RfRfg3 " "4!7!7!;!*dkk.H.H!H+//1/99"=GGVYYZgZnZno)//2^5I5I!5LL+,2248L8L8NNHHXXcdtcuv  "!r$   r   position_idsr*   r   r   labels	use_cacheoutput_attentionsoutput_hidden_statesreturn_dictkwargsreturnc                    |du |duz  rt        d      ||n| j                  j                  }||n| j                  j                  }||n| j                  j                  }|duxr |	du}|R| j                  j
                  | j                  k\  r/|| j                  j
                  k(  }|j                         }d||<   n|}| | j                         |      }|F||j                         nd}t        j                  |||j                  d   z   |j                        }||j                  d      dz   }|]| j                  |      }|j!                  |j                  |j"                        }| j%                  |||      }|j'                  ||      }| j)                  ||||||      } | j*                  d
|||||
||d|d	|}t-        |j.                  |j0                  |j2                  |j4                  |	      S d	      S )i  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, PaliGemmaForConditionalGeneration

        >>> model = PaliGemmaForConditionalGeneration.from_pretrained("google/paligemma2-3b-mix-224")
        >>> processor = AutoProcessor.from_pretrained("google/paligemma2-3b-mix-224")

        >>> prompt = "Where is the cat standing?"
        >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs,)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Where is the cat standing?\nsnow"
        ```Nz:You must specify exactly one of input_ids or inputs_embedsr   r   r   )r   rA   T)	r   r   r*   r   r   r   r   r   r   )r   r*   r+   r,   r   r#   )r   r1   r   r   use_return_dictr   rh   r   rl   get_seq_lengthr    r   r   r   r   r   r   r   r   masked_scatterr   ra   r   r   r*   r+   r,   )r=   r   r   r   r   r*   r   r   r   r   r   r   r   r   r   ry   r   llm_input_idspast_seen_tokensrA   r   outputss                         r%   rB   zPaliGemmaModel.forward  sd   ^ -t";<YZZ1B1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]$D0GV45G  T[[%?%?4??%R!*dkk.H.H!H%OO-M01M,-%M 7D557FM!CRC^==?de"\\ "2]5H5H5K"KTaThThN )33A6:L #!44\BN+..}/C/C]EXEXYN!%!:!:~ "; " *889K^\M..NO^]\g
 &$%% 
&%+'/!5)
 
 ,%77#33!//))2>2J
 	

 QU
 	
r$   )NNNNN)NNNNNNNNNNNNN)r   r   r   _checkpoint_conversion_mappingaccepts_loss_kwargsr   r6   rl   ro   rv   rx   r   boolr   r    r!   r   
LongTensorr   r   r   r   r   r-   r   r   r   r.   r   rB   rC   rD   s   @r%   r`   r`      s    '=>N%O"
 
:8&# &*@ d^@Du/@/@  "))":?:K:K"]b]n]n"0  '+*.1537KO595959-1$(,0/3&*k
##k
 ''k
 !.	k

 u//0k
 "%U->->(?(F"GHk
 !!1!12k
 !!1!12k
   1 12k
 ))*k
 D>k
 $D>k
 'tnk
 d^k
 -.k
  
u22	3!k
  k
r$   r`   z|
    The Base Paligemma model which consists of a vision backbone and a language model without language modeling head.,
    c            %           e Zd ZdddddZdgZdef fdZd	 Zd
 Zd Z	d Z
d Zed        Zed        Zed        Zee	 	 	 	 	 	 	 	 	 	 	 	 	 	 d(dej&                  dej(                  deej,                     deej&                     deeeej(                     ef      deej&                     deej&                     deej(                     deej&                     dee   dee   dee   dee   deeej,                  f   dee   d eeef   f d!              Z 	 	 	 	 	 	 	 	 	 	 d) fd"	Z!e"dej,                  d#ed$ed%ejF                  dej,                  d&efd'       Z$ xZ%S )*!PaliGemmaForConditionalGenerationzmodel.language_modelzmodel.vision_towerzmodel.multi_modal_projectorlm_head)z^language_model.modelz^vision_towerz^multi_modal_projectorz^language_model.lm_headzlm_head.weightr1   c                     t         |   |       t        |      | _        t	        j
                  |j                  j                  |j                  j                  d      | _	        | j                          y )NFr3   )r5   r6   r`   modelr   r7   rg   r9   rh   r   rj   r<   s     r%   r6   z*PaliGemmaForConditionalGeneration.__init__  sS     #F+
yy!3!3!?!?ASASA^A^ejkr$   c                 6    | j                   j                         S r@   )r   rl   rm   s    r%   rl   z6PaliGemmaForConditionalGeneration.get_input_embeddings  s    zz..00r$   c                 :    | j                   j                  |       y r@   )r   ro   rp   s     r%   ro   z6PaliGemmaForConditionalGeneration.set_input_embeddings  s    

''.r$   c                 :    | j                   j                  |       y r@   )r   rv   rt   s     r%   rv   z-PaliGemmaForConditionalGeneration.set_decoder  s    

w'r$   c                 6    | j                   j                         S r@   )r   rx   rm   s    r%   rx   z-PaliGemmaForConditionalGeneration.get_decoder  s    zz%%''r$   c                 8    | j                   j                  |      S r@   )r   r   )r=   r   s     r%   r   z4PaliGemmaForConditionalGeneration.get_image_features  s    zz,,\::r$   c                 .    | j                   j                  S r@   )r   ra   rm   s    r%   ra   z0PaliGemmaForConditionalGeneration.language_model  s    zz(((r$   c                 .    | j                   j                  S r@   )r   re   rm   s    r%   re   z.PaliGemmaForConditionalGeneration.vision_tower  s    zz&&&r$   c                 .    | j                   j                  S r@   )r   rf   rm   s    r%   rf   z7PaliGemmaForConditionalGeneration.multi_modal_projector  s    zz///r$   r   r   r   r   r*   r   r   r   r   r   r   r   r   logits_to_keepr   r   c                 >   ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  } | j                  d||||||||
|	||d|d|}|d   }t        |t              rt        | d      n|}| j                  |dd|ddf         }d}|	4 | j                  d||	| j                   j                  j                  d|}t        |||j                  |j                  |j                  |j                         S )r   NT)r   r   r   r   r   r*   r   r   r   r   r   r   r   r   )r)   r   rh   )r(   r)   r*   r+   r,   r   r#   )r1   r   r   r   r   rO   intslicer   loss_functionrg   rh   r'   r*   r+   r,   r   )r=   r   r   r   r   r*   r   r   r   r   r   r   r   r   r   r   r   r+   slice_indicesr)   r(   s                        r%   rB   z)PaliGemmaForConditionalGeneration.forward  sS   ^ 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]$** 
%))%+'/!5)
 
"  
8B>SV8W~ot4]kmA}a,?@A%4%% f9P9P9[9[_eD /#33!//)) ' ; ;
 	
r$   c                 \   t        |   |f||||||	|
|d|}|j                  d      |dxx   dz  cc<   |d   dk(  r||d<   |d uxr |d u}t        |t              xr t        |j                        }|d   dk(  r-|r+||n|}| j                  j                  ||||||      }||d<   |S )N)r*   r   r   r   r   r   r   r   r   r   r   r   r   )	r5   prepare_inputs_for_generationgetrO   r	   any
is_slidingr   r   )r=   r   r*   r   r   r   r   r   r   r   r   r   r   model_inputsry   is_static_hybrid_cacher   r   r>   s                     r%   r   z?PaliGemmaForConditionalGeneration.prepare_inputs_for_generation  s      w<
+')%)))
 
 N+7(A-( !!+7L($D0GV45G!+O[!I!mcRaRlRlNm!!&<,9,E=9L**88Q]_jK .9L)*r$   r   r   r   
batch_sizec                    | | j                         dk(  r| }|S t        j                  |      j                  }t        j                  ||f|||j
                        }|dk7  rt        j                  |d      }|t        j                  ||j
                        |j                  dd      kD  z  }|ddddddf   j                  |ddd      }| |j                         }| j                  d   }	|ddddddd|	f   | ddddddf   j                  |j
                        z   }
|
dk(  }
|ddddddd|	f   j                  |
|      |ddddddd|	f<   |S )	aM  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        Nr|   r}   r   r   r   rc   r   )r   r    r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   s              r%   5_prepare_4d_causal_attention_mask_with_cache_positionzWPaliGemmaForConditionalGeneration._prepare_4d_causal_attention_mask_with_cache_position<  s   > %.*<*<*>!*C(K* ' E*..I** -0Ye\j\q\qK !##jjqA5<<n>S>STWeWmWmnprsWtttK%dD!Q&67>>z1bRTUK))//1,2226*1aL[L+@ANSTVZ\`bcScDdDgDg&&E    ,q05@Aq,;,AV5W5c5c )6Aq!\k\12 r$   )NNNNNNNNNNNNNr   )
NNNNNNNTNN)&r   r   r   r   _tied_weights_keysr   r6   rl   ro   rv   rx   r   propertyra   re   rf   r   r   r    r   r!   r   r   r   r-   r   r   r   r   r   r.   r'   rB   r   staticmethodr   r   rC   rD   s   @r%   r   r     s    "8-"?#,	&" ++ 1/((; ) ) ' ' 0 0  '+*.1537KO595959-1$(,0/3&*34V
##V
 ''V
 !.	V

 u//0V
 "%U->->(?(F"GHV
 !!1!12V
 !!1!12V
   1 12V
 ))*V
 D>V
 $D>V
 'tnV
 d^V
 c5<</0V
  +,!V
" 
u55	6#V
  V
v -^ 444 4 {{	4
 4 4 4r$   r   )r   rF   r`   )+r   dataclassesr   typingr   r   r    torch.utils.checkpointr   cache_utilsr   r	   
generationr
   modeling_flash_attention_utilsr   modeling_outputsr   modeling_utilsr   processing_utilsr   utilsr   r   r   r   r   autor   configuration_paligemmar   
get_loggerr   loggerr   r'   Moduler0   rF   r`   r   __all__r#   r$   r%   <module>r      s5    ! "    - ) B 7 - &   4 
		H	% 
<#: < <  
<k < <2299  ) ) )0 
v
- v

v
r 
j(@/ j
jZ ^r$   