hdZddlZddlZddlmZddlmZmZmZddl Z ddl Z ddl m Z ddl m Z mZmZddlmZmZdd lmZmZmZdd lmZdd lmZdd lmZmZmZmZm Z m!Z!m"Z"m#Z#dd l$m%Z%ddl&m'Z'm(Z(m)Z)ddl*m+Z+m,Z,m-Z-ddl.m/Z/ddl0m1Z1e-jde3Z4dIdZ5Gdde jlZ7Gdde jlZ8Gdde jlZ9de8iZ:Gdde jlZ;Gdde jlZ<Gdd e jlZ=Gd!d"eZ>Gd#d$e jlZ?Gd%d&e jlZ@Gd'd(e jlZAe,Gd)d*e%ZBee,d+,Gd-d.e+ZCe,Gd/d0eBZDGd1d2e jlZEGd3d4e jlZFe,d5,Gd6d7eBZGe,d8,Gd9d:eBZHe,d;,Gd<d=eBZIe,d>,Gd?d@eBZJe,GdAdBeBZKe,GdCdDeBZLe,dE,GdFdGeBeZMgdHZNy)JzPyTorch ELECTRA model.N) dataclass)CallableOptionalUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FNget_activation)Cache DynamicCacheEncoderDecoderCache)GenerationMixin)GradientCheckpointingLayer)"BaseModelOutputWithCrossAttentions)BaseModelOutputWithPastAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer) ModelOutputauto_docstringlogging)deprecate_kwarg) ElectraConfigc ddl}ddl}ddl}t jj|}tjd||jj|}g} g} |D]^\} } tjd| d| |jj|| } | j| | j| `t| | D]\} } | } t|t r| j#dd} |d k(r$| j#d d } | j#d d } | j#d d} | j#dd} | j%d} t'd| Drtjd||}| D]}|j)d|r|j%d|}n|g}|ddk(s|ddk(r t+|d}nV|ddk(s|ddk(r t+|d}n9|ddk(r t+|d}n$|ddk(r t+|d}nt+||d}t-|dk\st/|d }||}j1d!r t+|d}n|dk(r|j3| } |j4| j4k7r&t7d"|j4d#| j4d$ t;d%| |t=j>| |_ |S#t$rtj dwxYw#t6$r1}|xj8|j4| j4fz c_d}~wwxYw#tB$r}t;d|| |Yd}~d}~wwxYw)&z'Load tf checkpoints in a pytorch model.rNzLoading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.z&Converting TensorFlow checkpoint from zLoading TF weight z with shape zelectra/embeddings/zgenerator/embeddings/ generatorzelectra/zdiscriminator/z generator/dense_1dense_predictionz!generator_predictions/output_biaszgenerator_lm_head/bias/c3$K|]}|dv yw)) global_step temperatureN).0ns j/var/www/html/eduruby.in/venv/lib/python3.12/site-packages/transformers/models/electra/modeling_electra.py z-load_tf_weights_in_electra..]sE1166Ez Skipping z [A-Za-z]+_\d+z_(\d+)kernelgammaweight output_biasbetabiasoutput_weightssquad classifierr# _embeddingszPointer shape z and array shape z mismatchedzInitialize PyTorch weight )"renumpy tensorflow ImportErrorloggererrorospathabspathinfotrainlist_variables load_variableappendzip isinstanceElectraForMaskedLMreplacesplitany fullmatchgetattrlenintendswith transposeshape ValueErrorargsprinttorch from_numpydataAttributeError)modelconfigtf_checkpoint_pathdiscriminator_or_generatorr>nptftf_path init_varsnamesarraysnamerXarray original_namepointerm_name scope_namesnumes r0load_tf_weights_in_electrarr3sz ggoo01G KK8 BC''0I E F  e (l5'BC&&w5 T e  5&)6 e! 3 %!34||$9;RS)[8||J0@A||L*=<< +=>D<< CE]^D::c?DEEE i 78G +<< 0&9"$((9f"=K#)(Kq>X-Q71J%gx8G ^}4 A&8P%gv6G ^'77%gx8G ^w.%g|G{#q(k!n-C%clG# +$}-!'848# U+ ==EKK/$~gmm_DUV[VaVaUbbm%noo0 .tf5} E ++E2GLg6n LQ   Q   @ 7==%++66    Im_-tQ 7  sV K1B*MB0M9A M?L*M1 L M,M  MM M5M00M5c eZdZdZfdZ d deejdeejdeejdeejde dejf d Z xZ S) ElectraEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c>t|tj|j|j |j |_tj|j|j |_ tj|j|j |_ tj|j |j|_ tj|j|_|j#dt%j&|jj)ddt+|dd|_|j#d t%j.|j0j3t$j4 dy) N) padding_idxeps position_ids)r#F) persistentposition_embedding_typeabsolutetoken_type_idsdtype)super__init__r Embedding vocab_sizeembedding_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_bufferr\arangeexpandrSr|zerosrysizelongselfra __class__s r0rzElectraEmbeddings.__init__s1 !||F,=,=v?T?Tbhbubuv#%<<0N0NPVPePe#f %'\\&2H2H&J_J_%`"f&;&;AVAVWzz&"<"<=   ELL)G)GHOOPWXej  (/v7PR\']$  ekk$*;*;*@*@*B%**Ubg  input_idsr~ry inputs_embedspast_key_values_lengthreturncZ||j}n|jdd}|d}||jdd|||zf}|st|dr-|jddd|f}|j |d|} | }n:t j |t j|jj}||j|}|j|} || z} |jdk(r|j|} | | z } |j| } |j| } | S)Nrzr#r~rrdevicer})rryhasattrr~rr\rrrrrr|rrr) rrr~ryrr input_shape seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedr embeddingsrs r0forwardzElectraEmbeddings.forwardsH  #..*K',,.s3K ^  ,,Q0FVlIl0l-lmL  !t-.*.*=*=a*n*M'3J3Q3QR]^_R`bl3m0!A!&[ SWSdSdSkSk!l   00;M $ : :> J"%::  ' ': 5"&":":<"H  - -J^^J/ \\*- r)NNNNr) __name__ __module__ __qualname____doc__rrr\ LongTensor FloatTensorrUTensorr __classcell__rs@r0rtrtsQ .15593759&' 'E,,-'!!1!12'u//0 '   1 12 ' !$ ' 'rrtceZdZdfd Zeddd ddej deejdeejd eejdee d ee d eej d e ej fd Z xZ S)ElectraSelfAttentionct||j|jzdk7r2t |ds&t d|jd|jd|j|_t |j|jz |_|j|jz|_tj|j|j|_ tj|j|j|_ tj|j|j|_ tj|j|_|xs t#|dd|_|j$dk(s|j$d k(rF|j&|_tj(d |j&zd z |j|_|j,|_||_y) NrrzThe hidden size (z6) is not a multiple of the number of attention heads ()r|r} relative_keyrelative_key_queryr<r#)rr hidden_sizenum_attention_headsrrYrUattention_head_size all_head_sizerLinearquerykeyvaluerattention_probs_dropout_probrrSr|rrdistance_embedding is_decoder layer_idxrrar|rrs r0rzElectraSelfAttention.__init__s     : : :a ?PVXhHi#F$6$6#78 445Q8  $*#=#= #&v'9'9F( ' -zC $  ' '> 9T=Y=Y]q=q+1+I+ID (&(ll1v7U7U3UXY3Y[_[s[s&tD # ++"rpast_key_valuepast_key_values4.58new_nameversion hidden_statesattention_mask head_maskencoder_hidden_statesoutput_attentionscache_positionrc |j\}} } |j|} | j|d|j|jj dd} |du} |St |trA|jj|j} | r |j}n|j}n|}| r|n|}| rK|I rGj|jj}|j|jj}n|j!|}|j|d|j|jj dd}|j#|}|j|d|j|jj dd}|D| s|nd}j%|||jd|i\}}| rd|j|j<t'j(| |j dd}|j*dk(s|j*dk(r| jd|jd}}|Dt'j,|dz t&j.|j0 jdd}n@t'j2|t&j.|j0 jdd}t'j2|t&j.|j0 jdd}||z }|j5||j6zdz }|j9| j: }|j*dk(rt'j<d | |}||z}nE|j*dk(r6t'j<d | |}t'j<d ||}||z|z}|t?j@|jz }|||z}tBjDjG|d }|jI|}|||z}t'j(||}|jKddddjM}|jOdd|jPfz}|j|}||fS)Nrzr#r<rTrrrrzbhld,lrd->bhlrzbhrd,lrd->bhlrdimrr ))rXrviewrrrWrMr is_updatedgetrcross_attention_cacheself_attention_cachelayerskeysvaluesrrupdater\matmulr|tensorrrrrrtoreinsummathsqrtr functionalsoftmaxrpermute contiguousrr)rrrrrrrr batch_sizer_ query_layeris_cross_attentionrcurr_past_key_valuecurrent_states key_layer value_layerattention_scores query_length key_lengthposition_ids_lposition_ids_rdistancepositional_embeddingrelative_position_scoresrelative_position_scores_queryrelative_position_scores_keyattention_probs context_layernew_context_layer_shapes r0rzElectraSelfAttention.forwardsN%2$7$7! Jjj/ !&&z2t7O7OQUQiQijtt q 3$>  &/+>?,77;;DNNK %*9*O*O'*9*N*N'&5#2D.- /"=*+224>>BGGI-44T^^DKKK0I!z2t7O7OQUQiQijtt1I**^4K%**B 8 8$:R:Ri1o *7It)<)C)C{DNN=M~<^*& ;&AEO..t~~>!<< Y5H5HR5PQ  ' '> 9T=Y=Y]q=q'2'8'8';Y__Q=O*L*!&j1nEJJWdWkWk!l!q!q""'l%**UbUiUi!j!o!oprtu!v"\\*EJJ}OcOcdiijkmopN%6H#'#:#:8dFbFb;bef;f#g #7#:#:ARAR#:#S ++~=+0<<8H+Wk+l(#36N#N --1EE16>NP[]q1r./4||S"S%**+BC o--rNNNNNNFN)rrrrr"r\rrrrbooltuplerrrs@r0rrs#6%0A6R7;15=A+/,115d.||d.!!2!23d.E--. d. ((9(9: d. "% d.$D>d.!.d. u|| d.Sd.rrcneZdZfdZdej dej dej fdZxZS)ElectraSelfOutputc(t|tj|j|j|_tj |j|j|_tj|j|_ yNrw) rrrrrdenserrrrrrs r0rzElectraSelfOutput.__init__Ns` YYv1163E3EF f&8&8f>S>STzz&"<"<= rr input_tensorrcr|j|}|j|}|j||z}|SNrrrrrrs r0rzElectraSelfOutput.forwardT7 =1  ]3 }|'CD rrrrrr\rrrrs@r0rrM1> U\\RWR^R^rreagerceZdZdfd ZdZeddd ddejdeejd eejd eejdee d ee d eejd e ejfdZ xZS)ElectraAttentionct|t|j||||_t ||_t|_y)Nr|r) rrELECTRA_SELF_ATTENTION_CLASSES_attn_implementationrroutputset pruned_headsrs r0rzElectraAttention.__init__bsF 263N3NO $;  (/ Erc>t|dk(ryt||jj|jj|j \}}t |jj||j_t |jj||j_t |jj||j_ t |jj|d|j_ |jjt|z |j_|jj|jjz|j_ |j j||_y)Nrr#r)rTrrrrrrrrrrrrunion)rheadsindexs r0 prune_headszElectraAttention.prune_headsls u:? 7 49900$))2O2OQUQbQb u -TYY__eD *499==%@ ,TYY__eD .t{{/@/@%QO )- (E(EE (R %"&))"?"?$))B_B_"_  --33E:rrrrrrrrrrrrc r|j|||||||}|j|d|} | f|ddz} | S)Nrrrrrrrr#)rr) rrrrrrrr self_outputsattention_outputoutputss r0rzElectraAttention.forward~s\yy )"7+/)!  ;;|A F#% QR(88rrr)rrrrrr"r\rrrrrrrrrs@r0rras";$%0A6R7;15=A+/,115||!!2!23E--.  ((9(9:  "% $D>!. u|| SrrcVeZdZfdZdej dej fdZxZS)ElectraIntermediatect|tj|j|j |_t|jtrt|j|_ y|j|_ yr) rrrrrintermediate_sizerrM hidden_actstrr intermediate_act_fnrs r0rzElectraIntermediate.__init__s] YYv1163K3KL f'' -'-f.?.?'@D $'-'8'8D $rrrcJ|j|}|j|}|Sr)rr()rrs r0rzElectraIntermediate.forwards& =1 00? rr rs@r0r#r#s#9U\\ellrr#cneZdZfdZdej dej dej fdZxZS) ElectraOutputc(t|tj|j|j |_tj|j |j|_tj|j|_ yr) rrrrr%rrrrrrrrs r0rzElectraOutput.__init__s` YYv779K9KL f&8&8f>S>STzz&"<"<= rrrrcr|j|}|j|}|j||z}|Srr r s r0rzElectraOutput.forwardr rr rs@r0r+r+r rr+c:eZdZdfd Zeddd ddej deejdeejd eejd eejdee d ee d eej d e ej fdZ dZ xZS) ElectraLayerclt||j|_d|_t |||_|j |_|j|_|jr-|j st|dt |d||_ t||_ t||_ y)Nr#rz> should be used as a decoder model if cross attention is addedr}r)rrchunk_size_feed_forward seq_len_dimr attentionradd_cross_attentionrYcrossattentionr# intermediater+r)rrarrs r0rzElectraLayer.__init__s '-'E'E$)&IF ++#)#=#=  # #?? D6)g!hii"26S]ir"sD /7#F+ rrrrrrrrrencoder_attention_maskrrrc H|j||||||} | d} | dd} |jrB|@t|dstd|d|j | ||||||} | d} | | ddz} t |j |j|j| } | f| z} | S)N)rrrrrrr#r6z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r) r4rrrYr6rfeed_forward_chunkr2r3)rrrrrr8rrrself_attention_outputsr r!cross_attention_outputs layer_outputs r0rzElectraLayer.forwards"& )/+) "0" 2!4(, ??4@4!12 =dVDDD '+&9&9 5#&; /"3-':' # 7q9  7 ;;G0  # #T%A%A4CSCSUe  /G+rcL|j|}|j||}|Sr)r7r)rr intermediate_outputr=s r0r:zElectraLayer.feed_forward_chunks,"//0@A{{#68HI rr)NNNNNFN)rrrrr"r\rrrrrrrr:rrs@r0r/r/s ,%0A6R7;15=A>B+/,115.||.!!2!23.E--. . ((9(9: . !)):): ; ."%.$D>.!.. u|| .S.`rr/cfeZdZdfd Z ddej deejdeejdeejdeejdeeeejdee d ee d ee d ee d eej d e eej e ffdZ xZ S)ElectraEncoderc t|||_tjt |j Dcgc]}t||c}|_d|_ ycc}w)Nr1F) rrrar ModuleListrangenum_hidden_layersr/layergradient_checkpointing)rrarirs r0rzElectraEncoder.__init__sQ  ]]uU[UmUmOn#o!L1$E#op &+#$psA%rrrrr8r use_cacheroutput_hidden_states return_dictrrc 4| rdnd} |rdnd} |r|jjrdnd}|jr%|jr|rtj dd}|rL|jj r6|4tt|jt|j}|rP|jj r:t|tr*tj dtj|}t|jD]W\}}| r| |fz} |||nd}||||||||| }|d}|s/| |dfz} |jjsO||d fz}Y| r| |fz} | std ||| | |fDSt||| | | S) Nr-zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)razPassing a tuple of `past_key_values` is deprecated and will be removed in Transformers v4.58.0. You should pass an instance of `EncoderDecoderCache` instead, e.g. `past_key_values=EncoderDecoderCache.from_legacy_cache(past_key_values)`.)r8rrrrr#r<c3$K|]}|| ywrr-)r.vs r0r1z)ElectraEncoder.forward..Ds = r2)last_hidden_staterr attentionscross_attentions)rar5rGtrainingrB warning_oncerrrrMrfrom_legacy_cache enumeraterFr)rrrrrr8rrIrrJrKrall_hidden_statesall_self_attentionsall_cross_attentionsrH layer_modulelayer_head_mask layer_outputss r0rzElectraEncoder.forwards#7BD$5b4%64;;;Z;Zr`d  & &4==##p" //O4K1,dkk2RT`hlhshsTtuO //JPU4V   \  2CCOTO(4 VOA|#$58H$H!.7.CilO(%'= /"3- M*!,M &9]1=MBEI$(,1/4&*15P ||P !!2!23P E--. P ((9(9: P !)):): ; P "%e.?.?(@"ABP D>P $D>P 'tnP d^P !.P  uU\\"$MM NP rrAc(eZdZdZfdZdZxZS)ElectraDiscriminatorPredictionszEPrediction module for the discriminator, made up of two dense layers.c t|tj|j|j|_t |j|_tj|jd|_ ||_ yNr#) rrrrrrr r& activationr(rars r0rz(ElectraDiscriminatorPredictions.__init__[s^ YYv1163E3EF ():):; " &*<*?  6 &&}5==bA rrrrrrrrrs@r0r]r]XsOrr]c(eZdZdZfdZdZxZS)ElectraGeneratorPredictionszAPrediction module for the generator, made up of two dense layers.ct|td|_t j |j |j|_t j|j|j |_ y)Ngelurw) rrr r`rrrrrrrrs r0rz$ElectraGeneratorPredictions.__init__nsV (0f&;&;AVAVWYYv1163H3HI rcl|j|}|j|}|j|}|Sr)rr`r)rgenerator_hidden_statesrs r0rz#ElectraGeneratorPredictions.forwardus3 #:;  6 }5 rrers@r0rgrgksKJrrgc*eZdZUeed<eZdZdZdZ y)ElectraPreTrainedModelraelectraTct|tjrm|jjj d|j j|j%|jjjyyt|tjrz|jjj d|j j|j2|jj|jjyyt|tjrJ|jjj|jjjdyy)zInitialize the weightsg)meanstdNg?)rMrrr5r^normal_rainitializer_ranger8zero_rrvrfill_)rmodules r0 _init_weightsz$ElectraPreTrainedModel._init_weightss fbii ( MM   & &CT[[5R5R & S{{&   &&('  - MM   & &CT[[5R5R & S!!- ""6#5#56<<>.  - KK   " " $ MM   $ $S ).rN) rrrr$__annotations__rrload_tf_weightsbase_model_prefixsupports_gradient_checkpointingrwr-rr0rmrm}s 0O!&*#*rrmz3 Output type of [`ElectraForPreTraining`]. ) custom_introceZdZUdZdZeejed<dZ eejed<dZ ee ejed<dZ ee ejed<y)ElectraForPreTrainingOutputa+ loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): Total loss of the ELECTRA objective. logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Prediction scores of the head (scores for each token before SoftMax). NlossrdrrP) rrrrrrr\rrxrdrrrPr-rr0r~r~sg)-D(5$$ %,*.FHU&& '.8cd d  "  6 6y. Q#..*K  &',,.s3KTU U!, J%.%:!! @T@T!"  &"/59 "1%++B/$335 #  !"ZZ FCN  !t(89*.//*H*HKZK*X'3J3Q3QR\^h3i0!A!&[ SY!Z"&"B"B>S^"_ ;; ! !&;&G=R=W=W=Y :  7$68O#P %-).4HQW)X&.2.H.HI_.` +.2 +&&y$++2O2OP %)'#9 (  4- . 33MBM 2"7#B+/!5#%  r) NNNNNNNNNNNNN)rrrrrrrr rr\rlistrrrrrrrrs@r0rrsk /0C-11515/3,0048<9==A$(,0/3&*\ELL)\!.\!. \ u||, \ ELL) \  -\ ( 5\!) 6\"$u'8'8"9:\D>\$D>\'tn\d^\ uU\\"$FF G\\rrc(eZdZdZfdZdZxZS)ElectraClassificationHeadz-Head for sentence-level classification tasks.czt|tj|j|j|_|j |j n |j}td|_ tj||_ tj|j|j|_ y)Nri)rrrrrrclassifier_dropoutrr r`rr num_labelsout_projrrarrs r0rz"ElectraClassificationHead.__init__(s YYv1163E3EF )/)B)B)NF % %TZTnTn )0zz"45  &"4"4f6G6GH rc |dddddf}|j|}|j|}|j|}|j|}|j|}|S)Nr)rrr`r)rfeatureskwargsxs r0rz!ElectraClassificationHead.forward2sZ Q1W  LLO JJqM OOA  LLO MM! rrers@r0rr%s7IrrceZdZdZdeffd Z ddejdeejdejfdZ xZ S) ElectraSequenceSummarya Compute a single vector summary of a sequence hidden states. Args: config ([`ElectraConfig`]): The config used by the model. Relevant arguments in the config class of the model are (refer to the actual config class of your model for the default values it uses): - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are: - `"last"` -- Take the last token hidden state (like XLNet) - `"first"` -- Take the first token hidden state (like Bert) - `"mean"` -- Take the mean of all tokens hidden states - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2) - `"attn"` -- Not implemented now, use multi-head attention - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction. - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes (otherwise to `config.hidden_size`). - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output, another string or `None` will add no activation. - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation. - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation. racft|t|dd|_|jdk(rtt j |_t|drq|jret|dr(|jr|jdkDr |j}n |j}t j|j||_t|dd}|r t|nt j |_t j |_t|dr3|j"dkDr$t j$|j"|_t j |_t|d r5|j(dkDr%t j$|j(|_yyy) N summary_typelastattnsummary_use_projsummary_proj_to_labelsrsummary_activationsummary_first_dropoutsummary_last_dropout)rrrSrNotImplementedErrorrIdentitysummaryrrrrrrr r` first_dropoutrr last_dropoutr)rra num_classesactivation_stringrs r0rzElectraSequenceSummary.__init__WsU #FNFC    && %{{} 6- .63J3Jv78V=Z=Z_e_p_pst_t$// $00 99V%7%7EDL#F,@$GIZN3D$E`b`k`k`m[[] 62 38T8TWX8X!#F,H,H!ID KKM 61 2v7R7RUV7V " 6+F+F GD 8W 2rr cls_indexrc|jdk(r |dddf}n|jdk(r |dddf}n|jdk(r|jd}n|jd k(r|Atj|d ddddf|jd dz tj }nX|j dj d}|jd |jdz z|jdfz}|jd |jd }n|jdk(rt|j}|j|}|j|}|j!|}|S)ak Compute a single vector summary of a sequence hidden states. Args: hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`): The hidden states of the last layer. cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*): Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token. Returns: `torch.FloatTensor`: The summary of the sequence hidden states. rNrzfirstrrpr#rr.rr)rzr)rrpr\ full_likerXr unsqueezerrrgatherrbrrrr`r)rrrrs r0rzElectraSequenceSummary.forwardtsn    &"1b5)F   ' )"1a4(F   & ("''A'.F   + - !OO!#rr1*-!''+a/** &//3==bA %,,Uimmo6I-JmN`N`acNdMf-fg "))"i8@@DF   & (% %##F+f%(""6* rr) rrrrr$rr\rrrrrrs@r0rr=sQ2H}H<Y])"..);CEDTDT;U)   )rrz ELECTRA Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. c^eZdZfdZe ddeejdeejdeejdeejdeejdeejdeejd eed eed eed e e eje ffd Z xZ S) ElectraForSequenceClassificationct|||j|_||_t ||_t ||_|jyr) rrrrarrnrr;rrs r0rz)ElectraForSequenceClassification.__init__sH   ++ #F+ 3F; rrrr~ryrrlabelsrrJrKrc | | n|jj} |j|||||||| |  } | d} |j| } d}||jj|j dk(rd|j_nl|j dkDrL|j tjk(s|j tjk(rd|j_nd|j_|jjdk(rIt}|j dk(r&|| j|j}n|| |}n|jjdk(r=t}|| jd|j |jd}n,|jjdk(rt}|| |}| s| f| ddz}||f|zS|St|| | j | j" S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nrr~ryrrrrJrKrr# regressionsingle_label_classificationmulti_label_classificationrzrrdrrP)rarrnr; problem_typerrr\rrUr rbr rrrrrP)rrrr~ryrrrrrJrKrcsequence_outputrdrloss_fctrs r0rz(ElectraForSequenceClassification.forwards(&1%D 'tnD d^D  uU\\"$<< =D D rrz Electra model with a binary classification head on top as used during pretraining for identifying generated tokens. It is recommended to load the discriminator checkpoint into that model. c^eZdZfdZe ddeejdeejdeejdeejdeejdeejdeejd eed eed eed e e eje ffd Z xZ S)ElectraForPreTrainingct||t||_t ||_|j yr)rrrrnr]discriminator_predictionsrrs r0rzElectraForPreTraining.__init__s3  #F+ )H)P& rrrr~ryrrrrrJrKrc d| | n|jj} |j|||||||| |  } | d} |j| } d}|t j }|a|j d| jddk(}| j d| jd|}||}|||j}n4|| j d| jd|j}| s| f| ddz}||f|zS|St|| | j| jS)am labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the ELECTRA loss. Input should be a sequence of tokens (see `input_ids` docstring) Indices should be in `[0, 1]`: - 0 indicates the token is an original token, - 1 indicates the token was replaced. Examples: ```python >>> from transformers import ElectraForPreTraining, AutoTokenizer >>> import torch >>> discriminator = ElectraForPreTraining.from_pretrained("google/electra-base-discriminator") >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-base-discriminator") >>> sentence = "The quick brown fox jumps over the lazy dog" >>> fake_sentence = "The quick brown fox fake over the lazy dog" >>> fake_tokens = tokenizer.tokenize(fake_sentence, add_special_tokens=True) >>> fake_inputs = tokenizer.encode(fake_sentence, return_tensors="pt") >>> discriminator_outputs = discriminator(fake_inputs) >>> predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2) >>> fake_tokens ['[CLS]', 'the', 'quick', 'brown', 'fox', 'fake', 'over', 'the', 'lazy', 'dog', '[SEP]'] >>> predictions.squeeze().tolist() [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0] ```Nrrrzr#r) rarrnrrrrrXfloatr~rrP)rrrr~ryrrrrrJrKrcdiscriminator_sequence_outputrdrr active_loss active_logits active_labelsrs r0rzElectraForPreTraining.forward spZ&1%Q 'tnQ d^Q  uU\\"$?? @Q Q rrz Electra model with a language modeling head on top. Even though both the discriminator and generator may be loaded into this model, the generator is the only model of the two to have been trained for the masked language modeling task. cpeZdZdgZfdZdZdZe ddee jdee jdee jdee jd ee jd ee jd ee jd ee d ee dee de e e jeffdZxZS)rNgenerator_lm_head.weightct||t||_t ||_t j|j|j|_ |jyr) rrrrnrggenerator_predictionsrrrrgenerator_lm_headrrs r0rzElectraForMaskedLM.__init__isR  #F+ %@%H"!#6+@+@&BSBS!T rc|jSrrrs r0get_output_embeddingsz(ElectraForMaskedLM.get_output_embeddingss%%%rc||_yrr)rrs r0set_output_embeddingsz(ElectraForMaskedLM.set_output_embeddingsvs !0rrrr~ryrrrrrJrKrc | | n|jj} |j|||||||| |  } | d} |j| } |j | } d}|Pt j }|| jd|jj|jd}| s| f| ddz}||f|zS|St|| | j| jS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Nrrrzr#r) rarrnrrrr rrrrrP)rrrr~ryrrrrrJrKrkgenerator_sequence_outputprediction_scoresrrrs r0rzElectraForMaskedLM.forwardys(&1%! 667PQ 223DE  **,H-222t{{7M7MNPVP[P[\^P_`D'),CAB,GGF)-)9TGf$ Ev E$1??.99   rr)rrr_tied_weights_keysrrrr rr\rrrrrrrrs@r0rNrN^s55&1-11515/3,004)-,0/3&*4 ELL)4 !.4 !. 4 u||, 4 ELL) 4   -4 &4 $D>4 'tn4 d^4  uU\\"N2 34 4 rrNz Electra model with a token classification head on top. Both the discriminator and generator may be loaded into this model. c^eZdZfdZe ddeejdeejdeejdeejdeejdeejdeejd eed eed eed e e eje ffd Z xZ S)ElectraForTokenClassificationc`t|||j|_t||_|j |j n |j }tj||_ tj|j|j|_ |jyr)rrrrrnrrrrrrrr;rrs r0rz&ElectraForTokenClassification.__init__s   ++#F+ )/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ rrrr~ryrrrrrJrKrc | | n|jj} |j|||||||| |  } | d} |j| } |j | } d}|?  ')HFKKDOO1 'tn1 d^1  uU\\"$99 :1 1 rrceZdZUeed<dZfdZe ddee jdee jdee jdee jdee jd ee jd ee jd ee jd ee d ee dee de e e jeffdZxZS)ElectraForQuestionAnsweringrarnct|||j|_t||_t j |j|j|_|jyr) rrrrrnrrr qa_outputsrrs r0rz$ElectraForQuestionAnswering.__init__sS   ++#F+ ))F$6$68I8IJ rrrr~ryrrstart_positions end_positionsrrJrKrc &| | n|jj} |j||||||| | } | d} |j| }|j dd\}}|j dj }|j dj }d}||t|jdkDr|j d}t|jdkDr|j d}|jd}|jd|}|jd|}t|}|||}|||}||zdz }| s||f| ddz}||f|zS|St|||| j| jS) N)rr~ryrrrrJrr#rzr) ignore_indexr<)r start_logits end_logitsrrP)rarrnrrPrbrrTrclampr rrrP)rrrr~ryrrrrrrJrKrcrrdrr total_loss ignored_indexr start_lossend_lossrs r0rz#ElectraForQuestionAnswering.forward s&1%@ 'tn@ d^@  uU\\"$@@ A@ @ rrc^eZdZfdZe ddeejdeejdeejdeejdeejdeejdeejd eed eed eed e e eje ffd Z xZ S)ElectraForMultipleChoicect||t||_t ||_t j|jd|_ |jyr_) rrrrnrsequence_summaryrrrr;rrs r0rz!ElectraForMultipleChoice.__init__PsM  #F+ 6v >))F$6$6: rrrr~ryrrrrrJrKrc L| | n|jj} ||jdn|jd} |!|jd|j dnd}|!|jd|j dnd}|!|jd|j dnd}|!|jd|j dnd}|1|jd|j d|j dnd}|j |||||||| |  } | d} |j | }|j|}|jd| }d}|t}|||}| s|f| ddz}||f|zS|St||| j| jS)a[ input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nr#rzrrrr) rarrXrrrnrr;r rrrP)rrrr~ryrrrrrJrK num_choicesrcr pooled_outputrdreshaped_logitsrrrs r0rz ElectraForMultipleChoice.forwardZsX&1%G>SINN2y~~b'9:Y] M[Mg,,R1D1DR1HImqM[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei (   r=#5#5b#9=;M;Mb;Q R  '+ll ))%'/!5#'3 ' #6a8--o> / ++b+6  ')HOV4D%'*Eab*IIF)-)9TGf$ Ev E("5CC2==   rr)rrrrr rr\rrrrrrrrs@r0rrNs-11515/3,004)-,0/3&*X ELL)X !.X !. X u||, X ELL) X   -X &X $D>X 'tnX d^X  uU\\"$== >X X rrzS ELECTRA Model with a `language modeling` head on top for CLM fine-tuning. c"eZdZdgZfdZdZdZe ddee jdee jdee jdee jd ee jd ee jd ee jd ee jd ee jdee e jdee dee dee dee de ee jeffdZxZS)ElectraForCausalLMrc$t|||jstj dt ||_t||_tj|j|j|_ |jy)NzOIf you want to use `ElectraForCausalLM` as a standalone, add `is_decoder=True.`)rrrrBwarningrrnrgrrrrrr init_weightsrs r0rzElectraForCausalLM.__init__sj    NNl m#F+ %@%H"!#6+@+@&BSBS!T rc|jSrrrs r0rz(ElectraForCausalLM.get_output_embeddingsrrc||_yrr)rnew_embeddingss r0rz(ElectraForCausalLM.set_output_embeddingss !/rrrr~ryrrrr8rrrIrrJrKrc ||n|jj}| d} |j||||||||| | | | | }|d}|j|j |}d}| *|j || fd|jj i|}|s|f|ddz}||f|zS|St|||j|j|j|jS)a3 labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Example: ```python >>> from transformers import AutoTokenizer, ElectraForCausalLM, ElectraConfig >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-base-generator") >>> config = ElectraConfig.from_pretrained("google/electra-base-generator") >>> config.is_decoder = True >>> model = ElectraForCausalLM.from_pretrained("google/electra-base-generator", config=config) >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.logits ```NF) rr~ryrrrr8rrIrrJrKrrr#)rrdrrrPrQ) rarrnrr loss_functionrrrrrPrQ)rrrr~ryrrrr8rrrIrrJrKrr!rrlm_lossrs r0rzElectraForCausalLM.forwards.R&1%S $D>S 'tnS d^S " uU\\"$EE F#S S rr) rrNrrrrrrrmrr) discriminator)OrrrD dataclassesrtypingrrrr\torch.utils.checkpointrtorch.nnrr r activationsr r cache_utilsrrr generationrmodeling_layersrmodeling_outputsrrrrrrrrmodeling_utilsr pytorch_utilsrrrutilsrr r!utils.deprecationr"configuration_electrar$ get_loggerrrBrrModulertrrrrr#r+r/rAr]rgrmr~rrrrrrNrrrr__all__r-rr0r s !,, AA1CC)9   .ll9900   H %Od? ?FA.299A.J   !" 3ryy3n "))  BII C-CNW RYYW tbii&"))$*_**.  :+ :  :x)xxv 0`RYY`F P '=P  P f[ 2[ [ |H /H H V@ $:@ @ FO "8O O dd 5d d N i /i  i X r