
    h0                     4   d dl Z d dlmZ d dlmZmZ d dlZd dlZd dlmZm	Z	 ddl
mZmZ ddlmZ ddlmZ dd	lmZmZmZmZ dd
lmZmZmZ ddlmZmZmZ ddlmZ ddl m!Z!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5m6Z6  ejn                  e8      Z9 G d de%      Z:d Z;d Z< G d de	jz                        Z> G d de+      Z? G d de,      Z@ G d de$      ZA G d d e'      ZB G d! d"e)      ZC G d# d$e(      ZD G d% d&e&      ZE G d' d(e*      ZFe G d) d*e             ZG G d+ d,eG      ZH G d- d.e	jz                        ZI G d/ d0e	jz                        ZJ G d1 d2e	jz                        ZKee G d3 d4e                    ZL G d5 d6e	jz                        ZM G d7 d8e	jz                        ZN G d9 d:e2      ZO G d; d<e3      ZP G d= d>e0      ZQ G d? d@e.      ZR G dA dBe/      ZS G dC dDe1      ZT G dE dFeT      ZU G dG dHeTe      ZVg dIZWy)J    N)	dataclass)OptionalUnion)Tensornn   )CacheDynamicCache)GenerationMixin)create_causal_mask)BaseModelOutputWithPast,BaseModelOutputWithPoolingAndCrossAttentionsCausalLMOutputWithPastModelOutput)ModuleUtilsMixinPreTrainedModelget_parameter_dtype)auto_docstringcan_return_tuplelogging)deprecate_kwarg)OutputRecordercheck_model_inputs   )	EsmAttentionEsmEmbeddings
EsmEncoderEsmIntermediateEsmLayer	EsmOutput	EsmPoolerEsmSelfAttentionEsmSelfOutput)LlamaAttentionLlamaDecoderLayerLlamaMLPLlamaPreTrainedModelLlamaRMSNormLlamaRotaryEmbedding   )EvollaConfigSaProtConfigc                        e Zd Z fdZ xZS )EvollaSaProtEmbeddingsc                 2    t         |   |       d | _        y N)super__init__position_idsselfconfig	__class__s     g/var/www/html/eduruby.in/venv/lib/python3.12/site-packages/transformers/models/evolla/modular_evolla.pyr2   zEvollaSaProtEmbeddings.__init__C   s          )__name__
__module____qualname__r2   __classcell__r7   s   @r8   r.   r.   B   s    ! !r9   r.   c                 b    | j                  dd      \  }}t        j                  | |fd      S )Nr   dim)chunktorchcat)xx1x2s      r8   rotate_half_esmrI   I   s/    WWQBWFB99rc2YB''r9   c                     |d d d d d | j                   d   d d f   }|d d d d d | j                   d   d d f   }| |z  t        |       |z  z   S )N)shaperI   )rF   cossins      r8   apply_rotary_pos_emb_esmrO   N   sY    
aMaggbkM1$
%C
aMaggbkM1$
%CG*S011r9   c                        e Zd ZU dZej
                  ed<   def fdZd
dZ	dej
                  dej
                  de
ej
                  ej
                  f   fd	Z xZS )EvollaSaProtRotaryEmbeddingz
    Rotary position embeddings based on those in
    [RoFormer](https://huggingface.co/docs/transformers/model_doc/roformer). Query and keys are transformed by rotation
    matrices which depend on their relative positions.
    inv_freqrB   c                     t         |           ddt        j                  d|dt        j                        j                         |z  z  z  }|}| j                  d|       d | _        d | _        d | _	        y )N      ?i'  r   r   dtyperR   )
r1   r2   rD   arangeint64floatregister_buffer_seq_len_cached_cos_cached_sin_cached)r5   rB   rR   r7   s      r8   r2   z$EvollaSaProtRotaryEmbedding.__init__^   sl    %ELLC%++$N$T$T$VY\$\]^Z2#r9   c                 t   |j                   |   }|| j                  k7  s#| j                  j                  |j                  k7  r|| _        t	        j
                  |j                   |   |j                        j                  | j                        }t	        j                  || j                        }t	        j                  ||fd      j                  |j                        }|j                         d d d d d d f   | _        |j                         d d d d d d f   | _        | j                  | j                  fS )Ndevicer@   rA   )rL   r[   r\   r`   rD   rW   type_asrR   outerrE   torM   rN   r]   )r5   rF   seq_dimensionseq_lentfreqsembs          r8   _update_cos_sin_tablesz2EvollaSaProtRotaryEmbedding._update_cos_sin_tablesi   s    ''-( d***d.>.>.E.E.Q#*D QWW]3AHHEMMdmm\AKK4==1E))UEN366qxx@C"wwytQ)9:D"wwytQ)9:D!1!111r9   qkreturnc                 .   | j                  |d      \  | _        | _        t        || j                  | j                        j	                  |j
                        t        || j                  | j                        j	                  |j
                        fS )NrK   )rd   rU   )ri   r\   r]   rO   rc   rV   )r5   rj   rk   s      r8   forwardz#EvollaSaProtRotaryEmbedding.forwardy   s    -1-H-HZ\-H-]*$* %Q(8(8$:J:JKNNUVU\U\N]$Q(8(8$:J:JKNNUVU\U\N]
 	
r9   )r   )r:   r;   r<   __doc__rD   r   __annotations__intr2   ri   tuplern   r=   r>   s   @r8   rQ   rQ   U   sY     ll	 C 	 2 
 
%,, 
5u||A[;\ 
r9   rQ   c                       e Zd ZddZy)EvollaSaProtSelfAttentionNc                    t         j                  j                  |        || _        |j                  |j
                  z  dk7  r2t        |d      s&t        d|j                   d|j
                   d      |j
                  | _        t        |j                  |j
                  z        | _	        | j
                  | j                  z  | _
        t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        |j                  | _        |xs t#        |dd      | _        d | _        | j$                  dk(  s| j$                  d	k(  rG|j(                  | _        t        j*                  d
|j(                  z  dz
  | j                        | _        n*| j$                  dk(  rt/        | j                        | _        |j0                  | _        || _        d| _        | j0                  xr | | _        y )Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()position_embedding_typeabsoluterelative_keyrelative_key_queryr   r*   rotaryrA   rT   )r   Moduler2   r6   hidden_sizenum_attention_headshasattr
ValueErrorrq   attention_head_sizeall_head_sizeLinearquerykeyvalueattention_probs_dropout_probdropoutgetattrrx   rotary_embeddingsmax_position_embeddings	Embeddingdistance_embeddingrQ   
is_decoder	layer_idxscaling	is_causal)r5   r6   rx   r   is_cross_attentions        r8   r2   z"EvollaSaProtSelfAttention.__init__   s   
		4  : ::a?PVXhHi#F$6$6#7 8 445Q8 
 $*#=#= #&v'9'9F<V<V'V#W !558P8PPYYv1143E3EF
99V//1C1CDYYv1143E3EF
::'> (
'-zC
$ "&''>9T=Y=Y]q=q+1+I+ID(&(ll1v7U7U3UXY3Y[_[s[s&tD#))X5%@TE]E]%^D" ++"C1C-Cr9   )NNF)r:   r;   r<   r2    r9   r8   rt   rt      s     Dr9   rt   c                       e Zd Zy)EvollaSaProtSelfOutputNr:   r;   r<   r   r9   r8   r   r          r9   r   c                       e Zd Zy)EvollaSaProtAttentionNr   r   r9   r8   r   r      r   r9   r   c                       e Zd Zy)EvollaSaProtIntermediateNr   r   r9   r8   r   r      r   r9   r   c                       e Zd Zy)EvollaSaProtOutputNr   r   r9   r8   r   r      r   r9   r   c                       e Zd Zy)EvollaSaProtLayerNr   r   r9   r8   r   r      r   r9   r   c                       e Zd Zy)EvollaSaProtEncoderNr   r   r9   r8   r   r      r   r9   r   c                       e Zd Zy)EvollaSaProtPoolerNr   r   r9   r8   r   r      r   r9   r   c                   d    e Zd ZU eed<   dgZdZdZdZe	 e
edd      g e
edd      gdZd	 Zy
)EvollaSaProtPreTrainedModelr6   r   Tr*   	attention)index
layer_namecrossattention)hidden_states
attentionscross_attentionsc                    | j                   j                  }t        |t        j                        rY|j
                  j                  j                  d|       |j                  %|j                  j                  j                          yyt        |t        j                        rf|j
                  j                  j                  d|       |j                  2|j
                  j                  |j                     j                          yyt        |t        j                        rJ|j                  j                  j                          |j
                  j                  j                  d       yy)zInitialize the weights        meanstdNrT   )r6   initializer_range
isinstancer   r   weightdatanormal_biaszero_r   padding_idx	LayerNormfill_r5   moduler   s      r8   _init_weightsz)EvollaSaProtPreTrainedModel._init_weights   s    kk++fbii(MM&&CS&9{{&  &&( '-MM&&CS&9!!-""6#5#56<<> .-KK""$MM$$S) .r9   N)r:   r;   r<   r,   rp   _no_split_modules_supports_flash_attn_supports_sdpa_supports_attention_backendr   r   rt   _can_record_outputsr   r   r9   r8   r   r      sX    ,-N"& +%&?qU`ab4AJZ[
*r9   r   c                        e Zd Zdef fdZd Zd Zd Ze	 dde	e
j                     de	e
j                     deee
j                     ef   fd	       Z	 dded
ee   de
j"                  de
j$                  def
dZ xZS )EvollaSaProtProteinEncoderr6   c                 d    t         |   |       t        |      | _        t	        |      | _        y r0   )r1   r2   r.   
embeddingsr   encoderr4   s     r8   r2   z#EvollaSaProtProteinEncoder.__init__   s(     08*62r9   c                 .    | j                   j                  S r0   r   word_embeddingsr5   s    r8   get_input_embeddingsz/EvollaSaProtProteinEncoder.get_input_embeddings   s    ...r9   c                 &    || j                   _        y r0   r   r5   r   s     r8   set_input_embeddingsz/EvollaSaProtProteinEncoder.set_input_embeddings   s    */'r9   c                     |j                         D ]7  \  }}| j                  j                  |   j                  j	                  |       9 y)z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsr   layerr   prune_heads)r5   heads_to_pruner   headss       r8   _prune_headsz'EvollaSaProtProteinEncoder._prune_heads   sE    
 +002 	CLE5LLu%//;;EB	Cr9   	input_idsattention_maskrl   c                 N   |j                         }|\  }}|j                  }|t        j                  ||f|      }| j	                  ||      }| j                  ||      }| j                  ||      }	|	d   }
t        |
|	j                  |	j                  |	j                        S )Nr_   r   r   )r   r   )last_hidden_stater   r   r   )sizer`   rD   onesr   get_extended_attention_maskr   r   r   r   r   )r5   r   r   input_shape
batch_size
seq_lengthr`   inputs_embedsextended_attention_maskencoder_outputssequence_outputs              r8   rn   z"EvollaSaProtProteinEncoder.forward   s      nn&!,
J!!!"ZZ*j)A6RN)N["&"B"B>S^"_,,}E\,])!,;-)77&11,==	
 	
r9   r   r`   rV   c                 4   |t        |       }|j                         dk(  r| j                  j                  s|t	        j
                  dt               |j                         dk(  r|dddddddf   }nk|j                         dk(  r<| j                  j                  rt        j                  |||      }n*|ddddddf   }nt        d| d|j                   d      |j                  |      }d	|z
  t        j                  |      j                  z  }|S )
a  
        Makes broadcastable attention and causal masks so that future and masked tokens are ignored.

        Arguments:
            attention_mask (`torch.Tensor`):
                Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
            input_shape (`Tuple[int]`):
                The shape of the input to the model.

        Returns:
            `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`.
        Nr   zNThe `device` argument is deprecated and will be removed in v5 of Transformers.r   z!Wrong shape for input_ids (shape z) or attention_mask (shape rw   rU   rT   )r   rB   r6   r   warningswarnFutureWarningr   *create_extended_attention_mask_for_decoderr   rL   rc   rD   finfomin)r5   r   r   r`   rV   r   s         r8   r   z6EvollaSaProtProteinEncoder.get_extended_attention_mask  s"    ='-E""$)dkk.D.D!dfs
 1$&4Qa]&C#!Q& {{%%*:*e*e+' +9D$9I*J'3K=@[\j\p\p[qqrs  #:"<"<5"<"I#&)@#@EKKPUDVDZDZ"Z&&r9   r0   )NN)r:   r;   r<   r,   r2   r   r   r   r   r   rD   r   r   rr   r   rn   rq   r`   rY   r   r=   r>   s   @r8   r   r      s    3| 3
/0C  26
ELL)
 !.
 
uU\\"$PP	Q	
 
2 rv2'$2'38:2'GL||2'chcncn2'	2'r9   r   c                   &     e Zd Zd fd	Zd Z xZS )!EvollaSequenceCompressorAttentionc                 j   t         |           |dz  | _        || _        ||z  }t	        j
                  |      | _        t	        j
                  |      | _        t	        j                  ||d      | _	        t	        j                  ||dz  d      | _
        t	        j                  ||d      | _        y )N      Fr   r   )r1   r2   scaler   r   r   
norm_medianorm_latentsr   to_qto_kvto_out)r5   rB   dim_headr   	inner_dimr7   s        r8   r2   z*EvollaSequenceCompressorAttention.__init__E  s    t^

u$	,,s+LL-IIc959	YYsIM>
ii	3U;r9   c                 F   | j                  |      }| j                  |      }| j                  }| j                  |      }t	        j
                  ||fd      }| j                  |      j                  dd      \  }}|j                  |j                  d      |j                  d      |d      j                  dddd      }|j                  |j                  d      |j                  d      |d      j                  dddd      }|j                  |j                  d      |j                  d      |d      j                  dddd      }|| j                  z  }t	        j                  ||j                  dd            }	|	|	j                  dd	      j                         z
  }	|	j                   \  }
}}}t	        j"                  ||      j%                  |j&                        }|d
d
d
d
d
d
f   }|d
d
d
d
d
d
f   }||z  }|	j)                  d|z
  j+                         d      }	|	j-                  d      }t	        j                  ||      }|j                  dddd      }|j/                  |j                  d      |j                  d      d      }| j1                  |      S )z
        Args:
            x (torch.Tensor): image features
                shape (b, n1, D)
            latent (torch.Tensor): latent features
                shape (b, n2, D);  n2: num of latent tokens
        rK   rA   r   r@   r   r*   r   TrB   keepdimNg     )r   r   r   r   rD   rE   r   rC   viewr   permuter   matmul	transposeamaxdetachrL   r   rc   r`   masked_fillboolsoftmaxreshaper   )r5   rF   latentsmaskhrj   kv_inputrk   vsimbsnhskdokdr   mask_expones_expattnouts                      r8   rn   z)EvollaSequenceCompressorAttention.forwardR  sB    OOA##G,JJIIg99a\r2zz(#))2 * 
1 FF166!9affQiB/771aCFF166!9affQiB/771aCFF166!9affQiB/771aC

N ll1akk"b12CHHTH299;;99BSzz"c"%%dkk24q()aD()("ooq4xoo/6{{r{"ll4#kk!Q1% kk#((1+sxx{B7{{3r9   )@      r:   r;   r<   r2   rn   r=   r>   s   @r8   r   r   D  s    <) r9   r   c                   &     e Zd Zd fd	Zd Z xZS )EvollaFeedForwardc                    t         |           t        ||z        }t        j                  |      | _        t        j                  ||d      | _        t        j                         | _	        t        j                  ||d      | _
        y NFr   )r1   r2   rq   r   r   normr   fc1GELU
activationfc2)r5   rB   multr   r7   s       r8   r2   zEvollaFeedForward.__init__  s`    d
O	LL%	99S)%8'')99Y%8r9   c           	      ~    | j                  | j                  | j                  | j                  |                        S r0   )r"  r!  r  r  )r5   rF   s     r8   rn   zEvollaFeedForward.forward  s+    xx1(>?@@r9   )   r  r>   s   @r8   r  r  ~  s    9Ar9   r  c                   *     e Zd Zdef fdZd Z xZS )!EvollaSequenceCompressorResamplerr6   c           
         t         |           |j                  j                  }|j                  | _        t        j                  t        j                  | j
                  |      d      | _
        t        j                  g       | _        t        |j                        D ]g  }| j                  j                  t        j                  t!        ||j"                  |j$                        t'        ||j(                        g             i t        j*                  |j                        | _        t        j.                  ||j                        | _        y )NT)requires_grad)rB   r   r   )rB   r#  )r1   r2   protein_encoder_configr~   resampler_num_latentsnum_latentsr   	ParameterrD   randnr	  
ModuleListlayersrangeresampler_depthappendr   resampler_dim_headresampler_headsr  resampler_ff_multr   r  r   protein_projector)r5   r6   protein_repr_dim_r7   s       r8   r2   z*EvollaSequenceCompressorResampler.__init__  s   !88DD!77||EKK0@0@BR$ScghmmB'v--. 
	AKK9 06;T;T\b\r\r *.>VE]E]^		
	 LL!3!34	!#+;V=O=O!Pr9   c                 j   |j                   d   }|j                   \  }}t        j                  || j                        j	                  |j
                        }t        j                  ||fd      }t        j                  |      j	                  | j                  j
                        }| j                  d    |j                  ddd      z  }|j	                  |j                        }| j                  D ]  \  }	}
 |	|||      |z   } |
|      |z   } | j                  |      }| j                  |      S )Nr   r*   rA   r@   )rL   rD   r   r,  rc   r`   rE   r	  r   rV   r0  r7  r  )r5   embedsr
  br  r9  latent_maskr   r	  r  fftransformed_features               r8   rn   z)EvollaSequenceCompressorResampler.forward  s   LLO

AjjT%5%5699$++Fyy$,!4 zz!} 3 34,,t$tyyQ'::**V\\* 	,HD"67D1G;GkG+G	, #44W=yy,--r9   )r:   r;   r<   r+   r2   rn   r=   r>   s   @r8   r'  r'    s    Q| Q*.r9   r'  c                       e Zd ZU dZej
                  ed<   dZeej
                     ed<   dZ	ee
ej
                  df      ed<   dZee
ej
                  df      ed<   y)EvollaProteinEncoderModelOutputNsequence_compressor_outputr   .r   r   )r:   r;   r<   rB  rD   FloatTensorrp   r   r   r   rr   r   r   r9   r8   rA  rA    si     59 1 1859x 1 129=AM8E%"3"3S"89:A:>Ju00#567>r9   rA  c                   f     e Zd Zdef fdZedej                  dej                  fd       Z	 xZ
S )EvollaProteinEncoderr6   c                 z    t         |           t        |j                        | _        t        |      | _        y )Nr6   )r1   r2   r   r*  modelr'  sequence_compressor_resamplerr4   s     r8   r2   zEvollaProteinEncoder.__init__  s.    /v7T7TU
-NV\-]*r9   r   r   c                     | j                  ||      }|j                  }| j                  ||      }t        ||j                        S )Nr   )rB  r   )rH  r   rI  rA  )r5   r   r   kwargsprotein_outputprotein_embedssequence_reprs          r8   rn   zEvollaProteinEncoder.forward  sJ    iW'99::>>Z.'4,>>
 	
r9   )r:   r;   r<   r+   r2   r   rD   
LongTensorrC  rn   r=   r>   s   @r8   rE  rE    s?    ^| ^
 
!1!1 
5CTCT 
 
r9   rE  c                   ~     e Zd Z	 	 	 ddee   dee   dee   f fdZd Z eddd	      	 	 	 	 	 	 	 dd
       Z xZ	S )#EvollaSequenceAlignerCrossAttentionprotein_encoder_dimstructure_encoder_dimmsa_encoder_dimc                    t         |           |j                  | _        |j                  | _        | j                  dz  | _        t        | j                  | j                  z        | _        | j                  | j                  z  | _        |j                  }|j                  }|j                  }t        j                  | j                  | j                        | _        |Kt        j                  || j                        | _        t        j                  || j                        | _        nd | _        d | _        |Kt        j                  || j                        | _        t        j                  || j                        | _        nd | _        d | _        |Kt        j                  || j                        | _        t        j                  || j                        | _        nd | _        d | _        t)        | j                        | _        t        j,                  |      | _        t        j                  | j                  | j                  |      | _        t3        | j                  |      | _        t        j6                  t9        j:                  dg            | _        t        j6                  t9        j:                  dg            | _        y )Nr   r   r   ) r1   r2   r~   r   r   rq   r   r   $aligner_attention_probs_dropout_probaligner_enable_biasaligner_ffn_multr   r   r   key_proteinvalue_proteinkey_structurevalue_structurekey_msa	value_msaEvollaRMSNormattention_normDropoutr   out_projr  r>  r-  rD   tensorgate_attentiongate_ffw)	r5   r6   rR  rS  rT  r   enable_biasffn_multr7   s	           r8   r2   z,EvollaSequenceAlignerCrossAttention.__init__  s    	!--#)#=#= --t3
#&t'7'7$:R:R'R#S !558P8PP'-'R'R$00**YYt//1C1CD
*!yy)<d>P>PQD!#+>@R@R!SD#D!%D ,!#+@$BTBT!UD#%99-BDDVDV#WD !%D#'D &99_d6H6HIDLYY8J8JKDNDL!DN+D,<,<=zz">?		$"2"2D4D4D;W#D$4$4h? ll5<<+>?U\\3%%89r9   c	                    |||g}	|	D 
cg c]  }
|
|
	 }	}
|	st        d      t        j                  |	d      }	| j                  |      }| j	                  |      }| j
                  @| j                  4|j                  |      }| j                  |      }| j                  |      }nd}d}| j                  @| j                  4|j                  |      }| j                  |      }| j                  |      }nd}d}| j                  @| j                  4|j                  |      }| j                  |      }| j                  |      }nd}d}|||g}|D 
cg c]  }
|
|
	 }}
t        j                  |d      }|||g}|D 
cg c]  }
|
|
	 }}
t        j                  |d      }|j                         dd | j                  | j                  fz   } |j                  | j!                  dddd      }|j                         dd | j                  | j                  fz   } |j                  | j!                  dddd      }|j                         dd | j                  | j                  fz   } |j                  | j!                  dddd      }|| j"                  z  }|Mt        j$                  |j                  d      |j                  d            j                  |j&                        }|ddddddf   |	ddddddf   z  }t        j(                  ||j+                  dd	            }||j-                  dd
      j/                         z
  }|j1                  d|z
  j3                         t        j4                  |j6                        j8                        } t;        j<                  d      |      }t        j(                  ||      }|j!                  dddd      j?                         }|j                         dd	 | j@                  fz   } |j                  | }| jC                  |      }|S c c}
w c c}
w c c}
w )z
        query_states: text
        key_value_states: protein
        query_states: [bs, query_seq_len, dim]
        key_value_states: [bs, kv_seq_len, dim]
        query_attn_mask: [bs, query_seq_len]
        kv_attn_mask: [bs, kv_seq_len]
        Nz=At least one modality should be provided for cross attention.r*   rA   r@   r   r   r   rK   Tr   )"r   rD   rE   r`  r   rY  rZ  rc   r[  r\  r]  r^  r   r   r   r   r   r   r   r`   r  r  r  r  r  r  r   rV   r   r   Softmax
contiguousr   rb  )r5   query_statesprotein_key_value_statesstructure_key_value_statesmsa_key_value_statesquery_attn_maskprotein_kv_attn_maskstructure_kv_attn_maskmsa_kv_attn_maskkv_attn_maskr9  query_layerkey_layer_proteinvalue_layer_proteinkey_layer_structurevalue_layer_structurekey_layer_msavalue_layer_msa	key_layervalue_layernew_query_layer_shapenew_key_layer_shapenew_value_layer_shaper   attn_weightsattention_scoresattention_probscontext_layernew_context_layer_shapes                                r8   cross_attentionz3EvollaSequenceAlignerCrossAttention.cross_attention  si   * -.DFVW#/Aa1=AA\]]yy15)),7 jj-'D,>,>,J'?'B'B<'P$ $ 0 01I J"&"4"45M"N $"&)d.B.B.N)C)F)F|)T&"&"4"45O"P$($8$89S$T!"&$(!<<#(B#7#:#:<#H  LL)=>M"nn-ABO M"O&(;]K	 );1Q]Q;	;IIiQ/	*,A?S"-?Qq??ii3 + 0 0 23B 7$$$$;
 !
 'k&&(=>FFq!QPQR'nn.s3$$$$7
 
 #INN$78@@Aq!L	 + 0 0 23B 7$$$$;
 !
 'k&&(=>FFq!QPQR!DJJ. "#jj):):1)=|?P?PQR?STWWXdXkXklO(D!T)9:\!TSWYZJZ=[[||K1D1DR1LM#l&7&7B&7&M&T&T&VV'33%%'\5G5G)H)L)L
 -"**,-=> _kB%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S***,CDm4q BL < @s"   P5P5P:P:P?P?past_key_valuepast_key_values4.58new_nameversionc           
      ^   |z|j                   \  }}}|jt        j                  ||      j                  |	j                        |	j                  ||f      j                  z  j                  |j                        }nd }|z|j                   \  }}}|jt        j                  ||      j                  |	j                        |
j                  ||f      j                  z  j                  |j                        }nd }|z|j                   \  }}}|jt        j                  ||      j                  |	j                        |j                  ||f      j                  z  j                  |j                        }nd }|}||j                         s$||j                         s||j                         rz|}| j                  ||||||||      }t        j                  | j                        |z  }||z   }|}| j                  |      t        j                  | j                        z  }||z   }|S )N)r   )rk  rl  rm  rn  ro  rp  rq  rr  )rL   rD   r   rc   r`   expandTanyr  tanhrd  r>  re  )r5   rk  protein_kv_statesstructure_kv_statesmsa_kv_statesro  rp  rq  rr  protein_batch_maskstructure_batch_maskmsa_batch_maskr  r  protein_kv_seq_lenrB   structure_kv_seq_lenmsa_kv_seq_lenr   residuals                       r8   rn   z+EvollaSequenceAlignerCrossAttention.forwardu  sL     (*;*A*A'B"C#+JJr#5699:L:S:ST(//6H"5M/NPPQ"&--. %
 $( *,?,E,E)B$c%-JJr#78;;<N<U<UV*118Lb7Q1RTTU"(//0 '
 &*"$&3&9&9#B'JJr>2556H6O6OP$++."1E+FHHI"]))* !
  $$ */C/G/G/I#/4J4N4N4P).>.B.B.D$H 00*):+>%2 /%9'=!1 1 	M "JJt':':;mKM$}4M$H GGM2UZZ5NNM$}4Mr9   )NNNNNNNNNN)
r:   r;   r<   r   rq   r2   r  r   rn   r=   r>   s   @r8   rQ  rQ    s~     .2/3)-1: &c]1:  (}	1:
 "#1:fn` %0A6R "#!G SGr9   rQ  c                       e Zd Zy)r_  Nr   r   r9   r8   r_  r_    r   r9   r_  c                       e Zd Zy)EvollaRotaryEmbeddingNr   r   r9   r8   r  r    r   r9   r  c                       e Zd Zy)	EvollaMLPNr   r   r9   r8   r  r    r   r9   r  c                       e Zd Zy)EvollaAttentionNr   r   r9   r8   r  r    r   r9   r  c                       e Zd Zdedef fdZ eddd      	 	 	 	 	 	 	 	 	 	 	 	 ddej                  d	e	ej                  ej                  f   d
e
ej                     de
ej                     de
e   de
e   de
ej                     de
ej                     de
ej                     de
ej                     de
ej                     de
ej                     de
ej                     de
ej                     fd       Z xZS )EvollaDecoderLayerr6   r   c                     t         |   ||       |dz   t        |j                  |j                  z  d      z  dk(  rt        ||j                        | _        y y )Nr*   r   )rR  )r1   r2   maxnum_hidden_layersaligner_num_add_layersrQ  r~   adapterr5   r6   r   r7   s      r8   r2   zEvollaDecoderLayer.__init__  s[    +MS!9!9V=Z=Z!Z\]^^bcc>$*$6$6DL dr9   r  r  r  r  r   position_embeddingsr   r3   	use_cachecache_positionr  r  r  r  r  r  ro  c                    |}| j                  |      } | j                  d|||||||d|\  }}||z   }|}| j                  |      }| j                  |      }||z   }t	        | d      r| j                  |||	|
||||      }|S )N)r   r   r3   r  r  r  r  r  )rk  r  r  r  ro  r  r  r  r   )input_layernorm	self_attnpost_attention_layernormmlpr   r  )r5   r   r  r   r3   r  r  r  r  r  r  r  r  r  ro  rK  r  r9  s                     r8   rn   zEvollaDecoderLayer.forward  s    & !,,]; *4>> 	
')%+) 3	
 	
q !=0 !55mD/ =04# LL*"3$7+ /#5%9- ) 	M r9   )NNNFNNNNNNNN)r:   r;   r<   r+   rq   r2   r   rD   r   rr   r   rO  r	   r  rn   r=   r>   s   @r8   r  r    sb   |   %0A6R
 2637+/$)59486:04597;15265||5 #5<<#=>5 !.	5
 u//05 "%5 D>5 !!1!125 $ELL15 &ell35  -5 %U\\25 'u||45 !.5 "%,,/5 S5r9   r  c                   &    e Zd ZdZdZdZg dZd Zy)EvollaPreTrainedModelF)r  r'  rQ  c                    | j                   j                  }t        j                  | |       t	        |t
              rd|j                  j                          |j                  j                          |j                  j                  j                  j                  d       y t	        |t              r(|j                  j                  j                  d|       y y )NrT   r   r   )r6   r   r   r   r   rQ  rd  r   re  r`  r   r   r   r'  r	  r   r   s      r8   r   z#EvollaPreTrainedModel._init_weights  s    kk++%%dF3fAB!!'')OO!!#!!((--33C8 ABNN''Sc': Cr9   N)r:   r;   r<   r   _supports_flex_attnr   r   r   r   r9   r8   r  r    s!     "';r9   r  c            !           e Zd Zdef fdZd Zd Zee	 	 	 	 	 	 	 	 	 	 	 	 	 dde	j                  dee	j                     dee	j                     dee   d	ee	j                     d
ee   dee	j                     dee	j                     dee	j                     dee	j                     dee	j                     dee	j                     dee	j                     deeef   fd              Z xZS )EvollaModelr6   c           	      F   t         |   |       |j                  | _        |j                  | _        t        j                  | j                  |j                  | j                        | _        t        |      | _
        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t!        |j                  |j"                        | _        t'        |      | _        t+        |dd      | _        | j/                          y c c}w )NrG  )r6   r   )epsgradient_checkpointingF)r1   r2   pad_token_idr   
vocab_sizer   r   r~   embed_tokensrE  protein_encoderr/  r1  r  r  r0  r_  rms_norm_epsr  r  
rotary_embr   r  	post_initr  s      r8   r2   zEvollaModel.__init__(  s     !.. ++LL&:L:LdN^N^_36Bmm "'v'?'?!@
 	 #!'
 "&"4"4&:M:MN	/v>&-f6NPU&V#s   $Dc                     | j                   S r0   r  r   s    r8   r   z EvollaModel.get_input_embeddings=  s       r9   c                     || _         y r0   r  r   s     r8   r   z EvollaModel.set_input_embeddings@  s
    !r9   r   r   r3   r  r   r  r  protein_input_idsprotein_attention_maskstructure_feats	msa_featsr  r  rl   c                    |du |duz  rt        d      || j                  |      }|r|t        | j                        }|F||j	                         nd}t        j                  |||j                  d   z   |j                        }||j                  d      }d}d}|S|	Q| j                  ||	      }|j                  }t        j                  dg|j                  d   z  |j                        }t        | j                  ||||	      }|}| j                  ||      }| j                  D ]  } ||f||||||||
|||||d
|} | j!                  |      }t#        ||      }|S )a;  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence in structure-aware tokens. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.
        structure_feats (torch.FloatTensor):
            The input IDs for purely structure-based features. Should be of shape `(batch_size, structure_seq_length, structure_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        msa_feats (torch.FloatTensor):
            The input IDs for purely MSA-based features. Should be of shape `(batch_size, msa_seq_length, msa_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        structure_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely structure-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `structure_feats`. Dummpy input for now.
        msa_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely MSA-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `msa_feats`. Dummpy input for now.
        Nz:You must specify exactly one of input_ids or inputs_embedsrG  r   r*   r_   r   T)r6   input_embedsr   r  r  )r   r3   r  r  r  r  r  r  r  r  r  r  ro  )r   r  )r   r  r
   r6   get_seq_lengthrD   rW   rL   r`   	unsqueezer  rB  rc  r   r  r0  r  r   )r5   r   r   r3   r  r   r  r  r  r  r  r  r  r  rK  past_seen_tokensprotein_featsr  protein_outputscausal_maskr   r  decoder_layeroutputs                           r8   rn   zEvollaModel.forwardC  s   B -t";<YZZ  --i8M0*$++>O!CRC^==?de"\\ "2]5H5H5K"KTaThThN )33A6L!(-C-O"22+5 3 O ,FFM!&tf7H7N7Nq7Q.QZkZrZr!s(;;&))+
 & #oom\J![[ 	M)*) /#-$7"/$3'#5%9- . M	& 		-0(++
 r9   )NNNNNNNNNNNNN)r:   r;   r<   r+   r2   r   r   r   r   rD   rO  r   r   r	   rC  r  r   rr   r   rn   r=   r>   s   @r8   r  r  '  sw   | *!"  '+1537+/59$(598<9=7;157;15b##b !.b u//0	b
 "%b   1 12b D>b !!1!12b $E$4$45b !) 6b "%"3"34b E--.b 'u||4b !.b  
u--	.!b  br9   r  c                       e Zd Z fdZd Zd Zee	 	 	 	 	 	 	 ddej                  de
ej                     de
ej                     de
ej                     dej                  d	e
ej                     d
e
e   fd              Z xZS )EvollaForProteinText2Textc                     t         |   |       t        |      | _        |j                  | _        t        j                  |j                  | j                  d      | _        | j                          y r  )
r1   r2   r  rH  r  r   r   r~   lm_headr  r4   s     r8   r2   z"EvollaForProteinText2Text.__init__  sQ      (
 ++yy!3!3T__5Qr9   c                 6    | j                   j                         S r0   )rH  r   r   s    r8   r   z.EvollaForProteinText2Text.get_input_embeddings  s    zz..00r9   c                 8    | j                   j                  |      S r0   )rH  r   r   s     r8   r   z.EvollaForProteinText2Text.set_input_embeddings  s    zz..u55r9   r   r   r   labelsr  r  r  c           
          | j                   d||||||d|}	|	d   }
| j                  |
      }d}|  | j                  d||| j                  d|}t	        |||	j
                  |	j                  |	j                        }|S )a,  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.

        Example:

        ```python
        >>> from transformers import EvollaProcessor, EvollaForProteinText2Text
        >>> model = EvollaForProteinText2Text.from_pretrained("westlake/Evolla-10B-hf")
        >>> processor = EvollaProcessor.from_pretrained("westlake/Evolla-10B-hf")

        >>> protein_information = {
            "aa_seq": "your amino acid sequence",
            "foldseek": "your foldseek sequence",
        }
        >>> question = "What is the function of this protein?"
        >>> message = [
            {"role": "system", "content": "You are an AI expert that can answer any questions about protein."},
            {"role": "user", "content": question},
        ]

        >>> inputs = processor(proteins=[protein_information], messages_list=[message], return_tensors="pt", padding="longest")
        >>> outputs = model.generate(**inputs)

        >>> print(processor.batch_decode(outputs, skip_special_tokens=True))
        ```)r   r   r   r  r  r  r   N)logitsr  r  )lossr  r  r   r   r   )rH  r  loss_functionr  r   r  r   r   )r5   r   r   r   r  r  r  r  rK  outputsr   r  r  
lm_outputss                 r8   rn   z!EvollaForProteinText2Text.forward  s    T $** 
)'/#9
 
  
m,%4%%iVFtibhiD+#33!//))

 r9   r  )r:   r;   r<   r2   r   r   r   r   rD   rO  r   r   rC  r  rn   r=   r>   s   @r8   r  r    s    16  '+1559-1.29=$(?##? !.?   1 12	?
 ))*? !++? !) 6? D>?  ?r9   r  )r  r  r  )Xr   dataclassesr   typingr   r   rD   torch.utils.checkpointr   r   cache_utilsr	   r
   
generationr   masking_utilsr   modeling_outputsr   r   r   r   modeling_utilsr   r   r   utilsr   r   r   utils.deprecationr   utils.genericr   r   esm.modeling_esmr   r   r   r   r   r    r!   r"   r#   llama.modeling_llamar$   r%   r&   r'   r(   r)   configuration_evollar+   r,   
get_loggerr:   loggerr.   rI   rO   r}   rQ   rt   r   r   r   r   r   r   r   r   r   r   r  r'  rA  rE  rQ  r_  r  r  r  r  r  r  r  __all__r   r9   r8   <module>r     s     ! "    . ) /  U T 
 1 ?
 
 
  = 
		H	%!] !(
2*
")) *
Z!D 0 !DH	] 		L 		 		 		 		* 		 	 */ * *>_'!< _'D7 		 7 tA		 A'.		 '.T ?k ?  ?
299 
$l")) l^	L 		0 		 		n 	?* ?D;0 ;*@' @FP 5 Pf Pr9   