hn?dZddlmZmZmZddlZddlZddlmZddlm Z ddl m Z ddl m Z dd lmZmZdd lmZdd lmZdd lmZdd lmZmZddlmZddlmZmZmZm Z m!Z!m"Z"ddl#m$Z$ejJe&Z'GddejPZ)Gdde!Z*dZ+d$dZ,GddeZ-GddeZ.Gdde Z/Gdd e Z0Gd!d"eZ1gd#Z2y)%zPyTorch Cohere model.)CallableOptionalUnionN)nn)Cache)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)dynamic_rope_update)ALL_ATTENTION_FUNCTIONS)Unpack)TransformersKwargslogging)deprecate_kwarg)LlamaAttentionLlamaForCausalLMLlamaMLP LlamaModelLlamaRotaryEmbeddingeager_attention_forward) CohereConfigc&eZdZdfd ZdZxZS)CohereLayerNormct|tjt j ||_||_y)zcThe hidden size can be a tuple or an int. The tuple is used for QKNorm to normalize across head_dimN)super__init__r Parametertorchonesweightvariance_epsilon)self hidden_sizeepsbias __class__s g/var/www/html/eduruby.in/venv/lib/python3.12/site-packages/transformers/models/cohere/modular_cohere.pyr zCohereLayerNorm.__init__7s/ ll5::k#:; #c|j}|jtj}|j dd}||z j dj dd}||z tj ||jzz}|jjtj|z}|j|S)NT)keepdimr) dtypetor"float32meanpowrsqrtr%r$)r& hidden_states input_dtyper3variances r+forwardzCohereLayerNorm.forward=s#)) %((7 !!"d!3!D(--a055b$5G&-XH]H]=]1^^  u}}5 E  ,,r,)Ngh㈵>F)__name__ __module__ __qualname__r r9 __classcell__r*s@r+rr6s $ -r,rcDeZdZejedZy)CohereRotaryEmbeddingc.|jddddfjj|jddd}|dddddfj}t |j j tr/|j j dk7r|j j nd}tj|d5|j|jzjdd}tj|dd }|j|jz}|j|jz} dddj|j  j|j fS#1swYAxYw) Nrr.rmpscpuF) device_typeenabledrdimr0)inv_freqfloatexpandshape isinstancedevicetypestrr"autocast transposerepeat_interleavecosattention_scalingsinr1r0) r&x position_idsinv_freq_expandedposition_ids_expandedrDfreqsembrTrVs r+r9zCohereRotaryEmbedding.forwardHsD!MM$4-8>>@GG HZHZ[\H]_acde ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfk ^^ U C 5&,,.1F1L1L1NNYYZ[]^_E))%;C'')d444C'')d444C  5 vvAGGv$cff177f&;;;  5 5s BF  FN)r:r;r<r"no_gradr r9r,r+r@r@Gs$U]]_ < C/0G3w;q>C/0G ::E: "GJJUJ$; ;;r,ceZdZfdZxZS) CohereMLPcJt||tj|j|j d|_tj|j|j d|_tj|j |jd|_y)NF)r)) rr rLinearr'intermediate_size gate_projup_proj down_projr&configr*s r+r zCohereMLP.__init__ss  4#3#3T5K5KRWXyy!1!143I3IPUV 4#9#94;K;KRWXr,)r:r;r<r r=r>s@r+rprp~sYYr,rpc8eZdZdZddedeeffd Zeddd dd e jd e e je jfd ee jdee d ee jd eede e jee jffdZxZS)CohereAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrx layer_idxc*t||||j|_|jret|j|j f|j |_t|j|j f|j |_ yy)Nr'r() rr use_qk_normrnum_attention_headshead_dimlayer_norm_epsq_normnum_key_value_headsk_normr&rxr{r*s r+r zCohereAttention.__init__s| +!--   )#77GVMbMbDK*#77GVMbMbDK r,past_key_valuepast_key_values4.58new_nameversionr6position_embeddingsattention_maskcache_positionkwargsreturnc |jdd}g|d|j}|j|j|} |j |j|} |j |j|} |j r"|j| } |j| } | jdd} | jdd} | jdd} |\} } t| | | | \} } |'| | |d}|j| | |j|\} } t}|jjdk7rt |jj}||| | | |f|j"sdn |j$|j&d|\}}|j(g|dj+}|j-|}||fS)Nr.rr)rVrTreagerg)dropoutscaling)rLrq_projviewk_projv_projr~rrrRrnupdater{rrx_attn_implementationrtrainingattention_dropoutrreshape contiguouso_proj)r&r6rrrrr input_shape hidden_shape query_states key_states value_statesrTrV cache_kwargsattention_interface attn_output attn_weightss r+r9zCohereAttention.forwards$))#2. 88b8$--8 {{=166|D [[/44\B {{=166|D   ;;|4LZ0J#--a3 ))!Q/ #--a3 &S#7 jRUWZ#[ j  &#&snUL'6'='=j,X\XfXfht'u $J (? ;; + +w 6"9$++:Z:Z"[ $7      % $}}C$2H2HLL %  % ! \*k));;;;FFH kk+. L((r,N)NN)r:r;r<__doc__rrintr rr"Tensortupler LongTensorrr r9r=r>s@r+rzrzsG |  %0A6R ,059 1)||1)#5<<#=>1)!. 1) "% 1) !!1!12 1)-.1) u||Xell33 41)S1)r,rzceZdZdedeffd Zeddd ddejd e ejd e ejde e d e e d e ejd e e ejejfdeede ej e e ej ej fffdZxZS)CohereDecoderLayerrxr{ct||j|_t|||_t ||_t|j|j|_ y)N)rxr{r}) rr r'rz self_attnrpmlprrinput_layernormrs r+r zCohereDecoderLayer.__init__sR !--()LV$.F!,,]; %3T^^ & ')%+) 3 &  & "!HH]3 #::=NN r,)NNNFNN)r:r;r<rrr rr"rrrrboolrrr FloatTensorr9r=r>s@r+rrsl|ll%0A6R2637+/$)59KO.||.!..u//0 . "% . D> .!!1!12.&eELL%,,,F&GH.-.. u  (51B1BEDUDU1U+V"WW X.S.r,rc$eZdZdeffd ZxZS) CohereModelrxc &t||tjt |j Dcgc]}t ||c}|_t||_ t|j|j|_ ycc}w)N)rxr})rr r ModuleListrangenum_hidden_layersrlayersr@ rotary_embrr'rnormrs r+r zCohereModel.__init__so  mmDI&JbJbDc dy  2 d  0v>#1C1C&J_J_`  esB)r:r;r<rr r=r>s@r+rrsa|aar,rcneZdZfdZ ddeej deejdeej deee e ejfdeejdeej dee d ee d ee d eej d ee ejfd eedefdZxZS)CohereForCausalLMct||t||_|j|_|j |_yr)rr rmodel logit_scaletie_word_embeddingsrws r+r zCohereForCausalLM.__init__s8   ( !--#)#=#= r, input_idsrrXr inputs_embedslabelsroutput_attentionsoutput_hidden_statesrlogits_to_keeprrc ||n|jj}| | n|jj} |jd|||||||| | d | } | j}t | t r t| dn| }|j|dd|ddf}||jz}d}|*|jd|||jjd| }t||| j| j| jS)az labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >> from transformers import AutoTokenizer, CohereForCausalLM >> model = CohereForCausalLM.from_pretrained("CohereForAI/c4ai-command-r-v01") >> tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-v01") >> prompt = "Hey, are you conscious? Can you talk to me?" >> inputs = tokenizer(prompt, return_tensors="pt") >> # Generate >> generate_ids = model.generate(inputs.input_ids, max_length=30) >> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```N) rrrXrrrrrr)logitsr vocab_size)lossrrr6 attentionsr^)rxrrrlast_hidden_staterMrslicelm_headr loss_functionrr rr6r)r&rrrXrrrrrrrrroutputsr6 slice_indicesrrs r+r9zCohereForCausalLM.forwards+J2C1N-TXT_T_TqTq$8$D $++JjJj  ,64:: , )%+'/!5) ,  ,  11 8B>SV8W~ot4]k mA}a,?@A$***  %4%%pVFt{{OeOepiopD%#33!//))   r,) NNNNNNNNNNr)r:r;r<r rr"rrrrlistrrrrrr r9r=r>s@r+rrs8>151537KO59-1$(,0/35934H E,,-H !.H u//0 H "%tE4E4E/F(F"GH H   1 12 H ))*H D>H $D>H 'tnH !!1!12H c5<</0H +,H  H r,r)rrCoherePreTrainedModel)Nr)3rtypingrrrr"torch.utils.checkpointr cache_utilsrmodeling_flash_attention_utilsr modeling_layersr modeling_outputsr r modeling_rope_utilsr modeling_utilsrprocessing_utilsrutilsrrutils.deprecationrllama.modeling_llamarrrrrrconfiguration_coherer get_loggerr:loggerModulerr@rfrnrprzrrr__all__r^r,r+rs.,,  B9O65&00/   H %-bii-"<0<"<<YYA)nA)H737ta*aO (O d r,