hU BddlZddlZddlmZddlmZmZmZddlZddlm Z m Z ddl m Z ddl mZmZddlmZdd lmZdd lmZdd lmZdd lmZmZmZmZmZdd lmZm Z ddl!m"Z"m#Z#m$Z$m%Z%ddl&m'Z'ddl(m)Z)m*Z*ddl+m,Z,m-Z-m.Z.ddl/m0Z0ddl1m2Z2m3Z3ddl4m5Z5m6Z6dZ7Gdde jpZ9dZ:dZ;Gdde jpZ< d`de jpdejdejd ejd!eejd"e=d#e=d$eejd%e'e,fd&Z>Gd'd(e jpZ?Gd)d*e jpZ@Gd+d,e jpZAd-ZBGd.d/e jpZCGd0d1e jpZDGd2d3eZEGd4d5e jpZFGd6d7e jpZGe-Gd8d9e$ZHGd:d;eHZIGd<d=e jpZJGd>d?e jpZKGd@dAe jpZLee-GdBdCeZMGdDdEe jpZNGdFdGe jpZOedHGdIdJe jpZPGdKdLe jpZQGdMdNe jpZRdOZSdadPZTdQejdReUdSejfdTZVGdUdVe jpZWGdWdXeZXe-GdYdZe$ZYGd[d\eYZZGd]d^eYeZ[gd_Z\y)bN) dataclass)CallableOptionalUnion)Tensornn)ACT2FN)Cache DynamicCache)GenerationMixin)use_kernel_forward_from_hub)create_causal_mask)GradientCheckpointingLayer)"BaseModelOutputWithCrossAttentionsBaseModelOutputWithPast,BaseModelOutputWithPoolingAndCrossAttentionsCausalLMOutputWithPast ModelOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSModuleUtilsMixinPreTrainedModelget_parameter_dtype)Unpack) find_pruneable_heads_and_indicesprune_linear_layer)TransformersKwargsauto_docstringcan_return_tuple)deprecate_kwarg)OutputRecordercheck_model_inputs) EvollaConfig SaProtConfigc|j|j}tj|dj ||z}|j |zS)a Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. Args: x: torch.Tensor x: Returns: torch.Tensor r%dim)neinttorchcumsumtype_aslong) input_ids padding_idxmaskincremental_indicess h/var/www/html/eduruby.in/venv/lib/python3.12/site-packages/transformers/models/evolla/modeling_evolla.py"create_position_ids_from_input_idsr65sP << $ ( ( *D,,t3;;DADH  # # % 33c8eZdZdZfdZ ddZdZxZS)EvollaSaProtEmbeddingszV Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. ct|tj|j|j |j |_|jr1tj|j |j|_ nd|_ tj|j|_t|dd|_|j#dt%j&|j(j+dd|j |_|j dk(r;tj|j(|j |j,|_|j0|_|j2|_d|_y) N)r2epsposition_embedding_typeabsolute position_ids)r%F persistent)super__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsemb_layer_norm_before LayerNormlayer_norm_eps layer_normDropouthidden_dropout_probdropoutgetattrr=register_bufferr-arangemax_position_embeddingsexpandr2position_embeddings token_dropout mask_token_idr?selfconfig __class__s r5rDzEvollaSaProtEmbeddings.__init__Js2 !||F,=,=v?Q?Q_e_r_rs  ' ' ll6+=+=6CXCXYDO"DOzz&"<"<= '.v7PR\']$  ELL)G)GHOOPWXej  "..  ' ': 5')||..0B0BPTP`P`(D $$11#11 r7c|*|t||j}n|j|}||j|}|}|jr||j ||j k(jdd}d}||jdn|jd}||j k(jdj|z }|d|z zd|z ddddfz j|j}|jdk(r|j|} || z}|j|j|}|-||jdzj|j}|S)Nr@gQ?r%r>)r6r2&create_position_ids_from_inputs_embedsrIrW masked_fillrX unsqueezesumshapefloattodtyper=rVrM) rZr1attention_maskr? inputs_embeds embeddingsmask_ratio_train src_lengthsmask_ratio_observedrVs r5forwardzEvollaSaProtEmbeddings.forwardcs  $A)TM]M]^ #JJ=Y   00;M#    )"7#//d>P>P1P0[0[\^0_adeJ) 4B4N.,,R0T]TcTcdeTfK#,0B0B#B"G"G"K"Q"Q"SVa"a $,<(<=EXAXZ[]acgZg@hhll  J  ' ': 5"&":":<"H #&99J ?? &4J  %$~'?'?'CCGG HXHXYJr7c|jdd}|d}tj|jdz||jzdztj|j }|j dj|S)z We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. Args: inputs_embeds: torch.Tensor Returns: torch.Tensor Nr@r%rfdevicer)sizer-rSr2r0rprarU)rZrh input_shapesequence_lengthr?s r5r_z=EvollaSaProtEmbeddings.create_position_ids_from_inputs_embedss$((*3B/ %a.||   q /D4D4D"Dq"HPUPZPZcpcwcw %%a(// <.>.E.E.Q#*D  QWW]3AHHEMMdmm\AKK4==1E))UEN366qxx@C"wwytQ)9:D "wwytQ)9:D !1!111r7qkreturnc.|j|d\|_|_t||j|jj |j t||j|jj |j fS)Nr)rr)rrrrrerf)rZrrs r5rmz#EvollaSaProtRotaryEmbedding.forwards-1-H-HZ\-H-]*$* %Q(8(8$:J:J K N NUVU\U\ N ] $Q(8(8$:J:J K N NUVU\U\ N ]  r7)r|) rurvrwrxr-r__annotations__r,rDrtuplermryrzs@r5rrsY ll  C  2   %,, 5u||A[;\ r7rmodulequerykeyvaluergscalingrP head_maskkwargsc tj||jdd|z} t|drN|jdvr?|j d} tj | tj| jjdd} tj | tj| jjdd} | | z } |j| |jzdz }|j|j}|jd k(rtjd ||}nB|jd k(r3tjd ||}tjd ||}||z}| z} |#|ddddddd|j d f}| |z} tj j#| dtj$j|j} tj j'| ||j(} || |z} tj| |}|jddj+}|| fS)Nr|r r= relative_keyrelative_key_queryror@r%rrzbhld,lrd->bhlrrzbhrd,lrd->bhlrr)r*rf)ptraining)r-matmul transposehasattrr=rcrSr0rpviewdistance_embeddingrTrerfeinsumr functionalsoftmaxfloat32rPr contiguous)rrrrrgrrPrr attn_weights seq_lengthposition_ids_lposition_ids_rdistancepositional_embeddingrelative_position_scoresrelative_position_scores_queryrelative_position_scores_key causal_mask attn_outputs r5eager_attention_forwardrs#<<s}}Q':;gELv01f6T6TY7[[^ j > !$Q1o " o%=> #k1 ==((2U]](SVVW\WbWbcL==((6??([L#i/ ,,|U3K''1-88:K  $$r7ceZdZd fd Z d dej deejdeejdeejdeejdee de ej fd Z xZ S) EvollaSaProtSelfAttentioncvt|||_|j|jzdk7r2t |ds&t d|jd|jd|j|_t|j|jz |_|j|jz|_ tj|j|j|_ tj|j|j|_ tj|j|j|_|j|_|xs t#|dd|_d|_|j$dk(s|j$d k(rG|j(|_tj*d |j(zd z |j|_n*|j$d k(rt/|j |_|j0|_||_d|_|j0xr| |_y)Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r=r>rrr|r%rotaryr)r)rCrDr[rGnum_attention_headsr ValueErrorr,attention_head_size all_head_sizerLinearrrrattention_probs_dropout_probrPrQr=rotary_embeddingsrTrErr is_decoder layer_idxr is_causal)rZr[r=ris_cross_attentionr\s r5rDz"EvollaSaProtSelfAttention.__init__s     : : :a ?PVXhHi#F$6$6#78 445Q8  $*#=#= #&v'9'9F( ' -zC $"&  ' '> 9T=Y=Y]q=q+1+I+ID (&(ll1v7U7U3UXY3Y[_[s[s&tD #  ) )X 5%@TE]E]%^D " ++" C1C-Cr7 hidden_statesrgrencoder_hidden_statesencoder_attention_maskrrc |jdd\}}||d|jf} |j|j| j dd} |du} | r|n|} | r|n|}|j | j| j dd} |j | j| j dd}| |jdzz} |jdk(r|j| | \} } t}|jjdk7r[|jdvr0td|jjd |jd t|jj}||| | ||f|jsd n |j|j |d |\}}|j#||dj%}||fS) Nr@r%r|reagerrzESM z attention does not support z^ embeddings. Set attention explicitly to 'eager' with `model.set_attn_implementation('eager')`r^)rPrr)rcrrrrrrr=rrr[_attn_implementationrrrrPrreshaper)rZrrgrrrr batch_sizer hidden_shape query_layerrcurrent_states key_layer value_layerattention_interfacerrs r5rmz!EvollaSaProtSelfAttention.forward4s"/!4!4Sb!9 J"JD4L4LM jj/44\BLLQPQR 2$>2D.-3E/>HH^,11,?II!QO jj055lCMMaQRS "D$<$B 3)||3)!!2!233)E--. 3) ((9(9: 3) !)):): ; 3)+,3) u|| 3)r7rc$eZdZfdZdZxZS)EvollaSaProtSelfOutputct|tj|j|j|_tj |j|_yN) rCrDrrrGdenserNrOrPrYs r5rDzEvollaSaProtSelfOutput.__init__ksB YYv1163E3EF zz&"<"<= r7cT|j|}|j|}||z}|SrrrPrZr input_tensors r5rmzEvollaSaProtSelfOutput.forwardp. =1  ]3 % 4 r7rurvrwrDrmryrzs@r5rrj > r7rcBeZdZdfd ZdZ ddeefdZxZS)EvollaSaProtAttentionct|t||||_t ||_t |_tj|j|j|_ y)N)rrr;) rCrDrrZroutputset pruned_headsrrKrGrL)rZr[rrr\s r5rDzEvollaSaProtAttention.__init__xsS -f ^pq ,V4 Ef&8&8f>S>STr7c>t|dk(ryt||jj|jj|j \}}t |jj||j_t |jj||j_t |jj||j_ t |jj|d|j_ |jjt|z |j_|jj|jjz|j_ |j j||_y)Nrr%r))lenrrZrrrrrrrrrrunion)rZheadsindexs r5 prune_headsz!EvollaSaProtAttention.prune_headss u:? 7 49900$))2O2OQUQbQb u -TYY__eD *499==%@ ,TYY__eD .t{{/@/@%QO )- (E(EE (R %"&))"?"?$))B_B_"_  --33E:r7rc |j|}|j|f||||d|\}} |j||}|S)Nrgrrr)rKrZr) rZrrgrrrrhidden_states_lnr_s r5rmzEvollaSaProtAttention.forwards] >>-8"  )"7#9     Qkk+}= r7)NFrt) rurvrwrDrrrrmryrzs@r5rrws1U;*"# +,r7rcj|dzdtj|tjdz zzS)zz This is the gelu implementation from the original EVOLLA_SA_PROT repo. Using F.gelu yields subtly wrong results. g?rg@)r-erfmathsqrt)rs r5gelurs. s7cEIIa$))C.&899 ::r7cVeZdZfdZdej dej fdZxZS)EvollaSaProtIntermediatect|tj|j|j |_yr)rCrDrrrGintermediate_sizerrYs r5rDz!EvollaSaProtIntermediate.__init__s, YYv1163K3KL r7rrc>|j|}t|}|Sr)rr)rZrs r5rmz EvollaSaProtIntermediate.forwards  =1 ]+ r7rurvrwrDr-rrmryrzs@r5rrs$MU\\ellr7rc$eZdZfdZdZxZS)EvollaSaProtOutputct|tj|j|j |_tj|j|_ yr) rCrDrrr rGrrNrOrPrYs r5rDzEvollaSaProtOutput.__init__sB YYv779K9KL zz&"<"<= r7cT|j|}|j|}||z}|Srrrs r5rmzEvollaSaProtOutput.forwardrr7rrzs@r5rrrr7rc@eZdZfdZ ddeefdZdZxZS)EvollaSaProtLayerct||j|_d|_t ||_|j |_|j|_|jr,|j st|dt |d|_ t||_ t||_ tj|j |j"|_y)Nr%z> should be used as a decoder model if cross attention is addedT)rr;)rCrDchunk_size_feed_forward seq_len_dimr attentionradd_cross_attention RuntimeErrorcrossattentionr intermediaterrrrKrGrLrYs r5rDzEvollaSaProtLayer.__init__s '-'E'E$.v6 ++#)#=#=  # #??"dV+i#jkk"7SW"XD 4V<(0 f&8&8f>S>STr7rc |j|f||d|}|jr5|3t|dstd|d|j|f||||d|}|j |}|S)N)rgrrz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r)rrrAttributeErrorrfeed_forward_chunk) rZrrgrrrrattention_output layer_outputs r5rmzEvollaSaProtLayer.forwards*4>>  )    ??4@4!12$=dVD`` 3t22  -#&;'=      ../?@ r7cn|j|}|j|}|j||}|Sr)rKrr)rZrattention_output_lnintermediate_outputrs r5rz$EvollaSaProtLayer.feed_forward_chunks<"nn-=>"//0CD{{#68HI r7rt) rurvrwrDrrrmrryrzs@r5rrs2 U$"# !+,!Fr7rcDeZdZfdZe ddeefdZxZS)EvollaSaProtEncoderc0t|||_tjt |j Dcgc] }t|c}|_tj|j|j|_ d|_ ycc}w)Nr;F)rCrDr[r ModuleListrangenum_hidden_layersrlayerrKrGrLemb_layer_norm_aftergradient_checkpointing)rZr[rr\s r5rDzEvollaSaProtEncoder.__init__sn  ]]uVMeMeGf#g!$5f$=#gh $&LL1C1CI^I^$_!&+#$hsBrc t|jD]\}}|||nd} ||f|| ||d|}|jr|j|}t|S)Nr)last_hidden_state) enumerater)r*r) rZrrgrrrri layer_modulelayer_head_masks r5rmzEvollaSaProtEncoder.forwards~ )4 OA|.7.CilO(-)&;'=   M   $ $ 55mDM1MRRr7rt) rurvrwrDr!rrrmryrzs@r5r$r$s=,"# S+,SSr7r$cVeZdZfdZdej dej fdZxZS)EvollaSaProtPoolerct|tj|j|j|_tj |_yr)rCrDrrrGrTanh activationrYs r5rDzEvollaSaProtPooler.__init__$s9 YYv1163E3EF '')r7rrc\|dddf}|j|}|j|}|S)Nr)rr6)rZrfirst_token_tensor pooled_outputs r5rmzEvollaSaProtPooler.forward)s6+1a40 #56  6 r7r rzs@r5r3r3#s#$ U\\ellr7r3cdeZdZUeed<dgZdZdZdZe e e ddge e ddgdZ d Z y ) EvollaSaProtPreTrainedModelr[rTr%r)r layer_namer)r attentionscross_attentionsc|jj}t|tjrY|j j jd||j%|jj jyyt|tjrf|j j jd||j2|j j |jjyyt|tjrJ|jj j|j j jdyy)zInitialize the weightsr^meanstdNr)r[initializer_range isinstancerrweightdatanormal_biaszero_rEr2rKfill_)rZrrBs r5 _init_weightsz)EvollaSaProtPreTrainedModel._init_weightsBskk++ fbii ( MM   & &CS & 9{{&   &&('  - MM   & &CS & 9!!- ""6#5#56<<>.  - KK   " " $ MM   $ $S ).r7N)rurvrwr'r_no_split_modules_supports_flash_attn_supports_sdpa_supports_attention_backendrr#r_can_record_outputsrKr7r5r;r;2sX ,-N"&+%&?qU`ab 4AJZ [  *r7r;c eZdZdeffd ZdZdZdZe dde e jde e jde e e jeffd Z dde d e ed e j"d e j$de f d ZxZS)EvollaSaProtProteinEncoderr[cdt||t||_t ||_yr)rCrDr9rir$encoderrYs r5rDz#EvollaSaProtProteinEncoder.__init__Ss(  08*62 r7c.|jjSrrirIrZs r5get_input_embeddingsz/EvollaSaProtProteinEncoder.get_input_embeddingsXs...r7c&||j_yrrWrZrs r5set_input_embeddingsz/EvollaSaProtProteinEncoder.set_input_embeddings[s*/'r7c|jD]7\}}|jj|jj |9y)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrUr)rr)rZheads_to_pruner)rs r5 _prune_headsz'EvollaSaProtProteinEncoder._prune_heads^sE +002 CLE5 LL  u % / / ; ;E B Cr7r1rgrcN|j}|\}}|j}|tj||f|}|j ||}|j ||}|j ||} | d} t| | j| j| jS)Nrr1rg)rgr)r-rr=r>) rqrpr-onesriget_extended_attention_maskrUrrr=r>) rZr1rgrrrrrprhextended_attention_maskencoder_outputssequence_outputs r5rmz"EvollaSaProtProteinEncoder.forwardfs  nn& !, J!!  !"ZZ*j)A6RN)N[ "&"B"B>S^"_,,}E\,])!,;-)77&11,==   r7rrrprfc4| t|}|jdk(r|jjs|t j dt |jdk(r|dddddddf}nk|jdk(r<|jjrtj|||}n*|ddddddf}ntd|d|jd|j|}d |z tj|jz}|S) a Makes broadcastable attention and causal masks so that future and masked tokens are ignored. Arguments: attention_mask (`torch.Tensor`): Mask with ones indicating tokens to attend to, zeros for tokens to ignore. input_shape (`Tuple[int]`): The shape of the input to the model. Returns: `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`. Nr|zNThe `device` argument is deprecated and will be removed in v5 of Transformers.r z!Wrong shape for input_ids (shape z) or attention_mask (shape rrr)rr*r[rwarningswarn FutureWarningr*create_extended_attention_mask_for_decoderrrcrer-finfomin)rZrgrrrprfres r5rdz6EvollaSaProtProteinEncoder.get_extended_attention_masks" ='-E""$)dkk.D.D! dfs    1 $&4Qa]&C #    !Q &{{%%*:*e*e+'+9D$9I*J'3K=@[\j\p\p[qqrs #:"<"<5"<"I#&)@#@EKKPUDVDZDZ"Z&&r7rNN)rurvrwr'rDrYr\r`r$rr-rrrrrmr,rprdrdryrzs@r5rSrSRs3|3 /0C26 ELL) !.  uU\\"$PP Q   2rv2'$2'38:2'GL||2'chcncn2' 2'r7rSc&eZdZdfd ZdZxZS)!EvollaSequenceCompressorAttentioncjt||dz|_||_||z}t j ||_t j ||_t j||d|_ t j||dzd|_ t j||d|_ y)NrFrHr|) rCrDscalerrrK norm_media norm_latentsrto_qto_kvto_out)rZr*dim_headr inner_dimr\s r5rDz*EvollaSequenceCompressorAttention.__init__s t^  u$ ,,s+LL-IIc959 YYsIM> ii 3U; r7cF|j|}|j|}|j}|j|}t j ||fd}|j |jdd\}}|j|jd|jd|djdddd}|j|jd|jd|djdddd}|j|jd|jd|djdddd}||jz}t j||jdd} | | jdd jz } | j \} } } } t j"| | j%|j&}|d d d d d d f}|d d d d d d f}||z}| j)d|z j+d } | j-d}t j||}|jdddd}|j/|jd|jdd}|j1|S) z Args: x (torch.Tensor): image features shape (b, n1, D) latent (torch.Tensor): latent features shape (b, n2, D); n2: num of latent tokens rr)r|r@rr%r Tr*keepdimNg)rurvrrwr-r~rxr}rrqpermutertrramaxdetachrcrcrerpr`boolrrry)rZrlatentsr3hrkv_inputrvsimbsnhskdokdrcmask_expones_expattnouts r5rmz)EvollaSequenceCompressorAttention.forwardsB OOA ##G, JJ IIg 99a\r2zz(#)) 2* 1 FF166!9affQiB / 7 71a C FF166!9affQiB / 7 71a C FF166!9affQiB / 7 71a C  Nll1akk"b12CHHTH299;;99BSzz"c"%%dkk24q()aD()("ooq4xoo/6{{r{"ll4#kk!Q1%kk#((1+sxx{B7{{3r7)@rrzs@r5rqrqs  <) r7rqc&eZdZdfd ZdZxZS)EvollaFeedForwardct|t||z}tj||_tj ||d|_tj|_ tj ||d|_ yNFrs) rCrDr,rrKnormrfc1GELUr6fc2)rZr*multr{r\s r5rDzEvollaFeedForward.__init__s` d O LL% 99S)%8'')99Y%8r7c ~|j|j|j|j|Sr)rr6rr)rZrs r5rmzEvollaFeedForward.forwards+xx1(>?@@r7)rrzs@r5rrs9Ar7rc*eZdZdeffd ZdZxZS)!EvollaSequenceCompressorResamplerr[c t||jj}|j|_t jtj|j |d|_ t jg|_ t|jD]g}|jjt jt!||j"|j$t'||j(git j*|j|_t j.||j|_y)NT) requires_grad)r*rzr)r*r)rCrDprotein_encoder_configrGresampler_num_latents num_latentsr Parameterr-randnrr&layersr'resampler_depthappendrqresampler_dim_headresampler_headsrresampler_ff_multrKrrprotein_projector)rZr[protein_repr_dimrr\s r5rDz*EvollaSequenceCompressorResampler.__init__s !88DD!77||EKK0@0@BR$Scgh mmB' v--. A KK   9 06;T;T\b\r\r*.>VE]E]^   LL!3!34 !#+;V=O=O!Pr7cj|jd}|j\}}tj||jj |j }tj ||fd}tj|j |jj }|jd|jdddz}|j |j}|jD]\} } | ||||z}| ||z}|j|} |j| S)Nrr%r)r@) rcr-rcrrerpr~rrrfrrr) rZembedsr3brr latent_maskrcrrfftransformed_features r5rmz)EvollaSequenceCompressorResampler.forwards LLO AjjT%5%5699$++F yy$ ,!4zz!} 3 34,,t$tyyQ'::**V\\*  ,HD"67D1G;GkG+G ,#44W=yy,--r7)rurvrwr&rDrmryrzs@r5rrsQ|Q*.r7rceZdZUdZej ed<dZeej ed<dZ ee ej dfed<dZ ee ej dfed<y)EvollaProteinEncoderModelOutputNsequence_compressor_outputr-.rr=) rurvrwrr-rrr-rrrr=rQr7r5rr&si59 1 1859x 1 129=AM8E%"3"3S"89:A:>Ju00#567>r7rcfeZdZdeffd ZedejdejfdZ xZ S)EvollaProteinEncoderr[czt|t|j|_t ||_y)Nr[)rCrDrSrmodelrsequence_compressor_resamplerrYs r5rDzEvollaProteinEncoder.__init__0s. /v7T7TU -NV\-]*r7r1rgc |j||}|j}|j||}t||jS)Nrb)rr-)rr-rr)rZr1rgrprotein_outputprotein_embeds sequence_reprs r5rmzEvollaProteinEncoder.forward5sJiW'99::>>Z .'4,>>  r7) rurvrwr&rDr!r- LongTensorrrmryrzs@r5rr/s?^|^  !1!1 5CTCT  r7rc~eZdZ d deedeedeeffd ZdZeddd  d d ZxZ S) #EvollaSequenceAlignerCrossAttentionprotein_encoder_dimstructure_encoder_dimmsa_encoder_dimct||j|_|j|_|jdz|_t |j|jz |_|j|j z|_|j}|j}|j}tj|j|j|_ |Ktj||j|_tj||j|_nd|_d|_|Ktj||j|_tj||j|_nd|_d|_|Ktj||j|_tj||j|_nd|_d|_t)|j|_tj,||_tj|j|j||_t3|j||_tj6t9j:dg|_tj6t9j:dg|_y)Nrrsr^) rCrDrGrrtr,rr$aligner_attention_probs_dropout_probaligner_enable_biasaligner_ffn_multrrr key_protein value_protein key_structurevalue_structurekey_msa value_msa EvollaRMSNormattention_normrNrPout_projrrrr-tensorgate_attentiongate_ffw) rZr[rrrr enable_biasffn_multr\s r5rDz,EvollaSequenceAlignerCrossAttention.__init__Bs !--#)#=#= --t3 #&t'7'7$:R:R'R#S !558P8PP'-'R'R$00 **YYt//1C1CD  *!yy)P>PQD !#+>@R@R!SD #D !%D  ,!#+@$BTBT!UD #%99-BDDVDV#WD !%D #'D  &99_d6H6HIDLYY8J8JKDNDL!DN+D,<,<=zz">?  $"2"2D4D4D;W #D$4$4h? ll5<<+>? U\\3%%89 r7c |||g} | D cgc]} | |  } } | s tdtj| d} |j|} |j | } |j @|j 4|j|}|j |} |j |} nd} d} |j@|j4|j|}|j|}|j|}nd}d}|j@|j4|j|}|j|}|j|}nd}d}| ||g}|D cgc]} | |  }} tj|d}| ||g}|D cgc]} | |  }} tj|d}| jdd|j|jfz}| j|j!dddd} |jdd|j|jfz}|j|j!dddd}|jdd|j|jfz}|j|j!dddd}| |j"z} |Mtj$|jd|jdj|j&}|ddddddf| ddddddfz}tj(| |j+dd }||j-dd j/z }|j1d|z j3tj4|j6j8}t;j<d|}tj(||}|j!ddddj?}|jdd |j@fz}|j|}|jC|}|Scc} wcc} wcc} w) z query_states: text key_value_states: protein query_states: [bs, query_seq_len, dim] key_value_states: [bs, kv_seq_len, dim] query_attn_mask: [bs, query_seq_len] kv_attn_mask: [bs, kv_seq_len] Nz=At least one modality should be provided for cross attention.r%r)r@rr|r rTr})"rr-r~rrrrrerrrrrqrrrrrtrcrprrrrr`rrmrfrnrSoftmaxrrr)rZ query_statesprotein_key_value_statesstructure_key_value_statesmsa_key_value_statesquery_attn_maskprotein_kv_attn_maskstructure_kv_attn_maskmsa_kv_attn_mask kv_attn_maskrrkey_layer_proteinvalue_layer_proteinkey_layer_structurevalue_layer_structure key_layer_msavalue_layer_msarrnew_query_layer_shapenew_key_layer_shapenew_value_layer_shapergrattention_scoresattention_probs context_layernew_context_layer_shapes r5cross_attentionz3EvollaSequenceAlignerCrossAttention.cross_attentionusi*-.DFVW #/Aa1=A A\] ]yy15 )),7 jj-    'D,>,>,J'?'B'B<'P $ $ 0 01I J "&"4"45M"N  $ "&     )d.B.B.N)C)F)F|)T &"&"4"45O"P $($8$89S$T !"& $( ! << #(B#7#:#:<#H  LL)=>M"nn-ABO M"O&(;]K );1Q]Q; ;IIiQ/ *,A?S "-?Qq? ?ii 3 + 0 0 23B 7  $ $  $ $; ! 'k&&(=>FFq!QPQR 'nn.s3  $ $  $ $7  #INN$78@@Aq!L + 0 0 23B 7  $ $  $ $; ! 'k&&(=>FFq!QPQR !DJJ.   "#jj):):1)=|?P?PQR?STWWXdXkXklO(D!T)9:\!TSWYZJZ=[[||K1D1DR1LM #l&7&7B&7&M&T&T&VV '33   % % '\5G5G)H)L)L -"**,-=> _kB %--aAq9DDF "/"4"4"6s";t?Q?Q>S"S* **,CD  m4 qBL<@s"P5P5P:P:P? P?past_key_valuepast_key_values4.58new_nameversionc ^|z|j\} }}|jtj| |j| j| j || fj zj|j}nd}|z|j\} }}|jtj| |j| j| j || fj zj|j}nd}|z|j\} }}|jtj| |j| j| j || fj zj|j}nd}|}||js$||js||jrz|}|j||||||||}tj|j|z}||z}|}|j|tj|jz}||z}|S)N)rq)rrrrrrrr) rcr-rcrerprUTanyrtanhrrr)rZrprotein_kv_statesstructure_kv_states msa_kv_statesrrrrprotein_batch_maskstructure_batch_maskmsa_batch_maskrrprotein_kv_seq_lenr*structure_kv_seq_lenmsa_kv_seq_lenrresiduals r5rmz+EvollaSequenceAlignerCrossAttention.forwardsL  (*;*A*A 'B"C#+JJr#5699:L:S:ST(//6H"5M/NPPQ"&--.% $(  *,?,E,E )B$c%-JJr#78;;2556H6O6OP$++."1E+FHHI"]))*! $ $  */C/G/G/I#/4J4N4N4P).>.B.B.D$H 00*):+>%2 /%9'=!11 M"JJt':':;mKM$}4M$H GGM2UZZ 5NNM$}4Mr7)NNNNNNNNNN) rurvrwrr,rDrr"rmryrzs@r5rrAs~.2/3)- 1:&c]1: (} 1: "# 1:fn`%0A6R"#!GSGr7rRMSNormc,eZdZdfd ZdZdZxZS)rct|tjt j ||_||_y)z< EvollaRMSNorm is equivalent to T5LayerNorm N)rCrDrrr-rcrEvariance_epsilon)rZrGr<r\s r5rDzEvollaRMSNorm.__init__2s1 ll5::k#:; #r7c"|j}|jtj}|j dj dd}|tj ||jzz}|j|j|zS)Nr|r@T)r~) rfrer-rpowrArsqrtr rE)rZr input_dtypevariances r5rmzEvollaRMSNorm.forward:sy#)) %((7  $$Q',,R,>% Ht?T?T4T(UU {{]--k:::r7c^t|jjd|jS)Nz, eps=)rrErcr rXs r5 extra_reprzEvollaRMSNorm.extra_reprAs* ))*+6$2G2G1HIIr7)gư>)rurvrwrDrmrryrzs@r5rr0s$;Jr7rc~eZdZUejed<ddeffd Zeje dZ xZ S)EvollaRotaryEmbeddingrr[ct|t|drUt|jt r;|jj d|jj d|_nd|_|j|_ |j|_ ||_ t|j|_ |j|j|\}|_|jd|d|j |_y)N rope_scaling rope_typetypedefaultrFrA)rCrDrrDrdictgetrrTmax_seq_len_cachedoriginal_max_seq_lenr[r rope_init_fnattention_scalingrRroriginal_inv_freq)rZr[rprr\s r5rDzEvollaRotaryEmbedding.__init__Hs  6> *z&:M:Mt/T#0044[&BUBUBYBYZ`BabDN&DN"("@"@$*$B$B! /?+/+<+>@GG HZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfk ^^ U C 5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C  5 vvAGGv$cff177f&;;;  5 5s BF%%F.r) rurvrwr-rrr&rDno_gradrrmryrzs@r5rrEs=ll/|/"U]]_ <