hN`ddlmZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZmZmZddlZddlmZddlmZddlmZdd lmZmZdd lmZdd lmZej@Z!d Z"dd Z#ddZ$ddZ%e e&ee&e'e(e)ffZ*GddeZ+GddeZ,GddeZ-ddZ.ddZ/y)) annotationsN)sha1)Thread)AnyDictIterableListMappingOptionalTupleUnion)Document) Embeddings) VectorStore) BaseSettingsSettingsConfigDict) TypedDict)maximal_marginal_relevanceFc|D]}||vsyy)z Check if a string has multiple substrings. Args: s: The string to check *args: The substrings to check for in the string Returns: bool: True if all substrings are present in the string, False otherwise FT)sargsas h/var/www/html/eduruby.in/venv/lib/python3.12/site-packages/langchain_community/vectorstores/starrocks.pyhas_mul_sub_strrs" A: c(tr t|yy)z[ Print a debug message if DEBUG is True. Args: s: The message to print N)DEBUGprint)rs r debug_outputr &s   a rc*|j}|j||j}g}|jD]5}i}t |D]\}}||d} ||| <|j |7t ||j|S)z Get a named result from a query. Args: connection: The connection to the database query: The query to execute Returns: List[dict[str, Any]]: The result of the query r)cursorexecute descriptionfetchall enumerateappendr close) connectionqueryr"columnsresultvalueridxdatumks rget_named_resultr20s   F NN5  G F" #E* JC QAAaD   a   LLN Mrc@eZdZUded<ded<ded<ded<d ed <y ) QueryResultzList[List[str]]idsz List[Any] embeddingsList[Document] documentszOptional[List[Metadata]] metadataszOptional[List[float]] distancesN)__name__ __module__ __qualname____annotations__rrrr4r4Ls ''$$rr4ceZdZUdZdZded<dZded<dZded <d Zded <d d dddZ ded<dZ ded<dZ ded<ddZ e ddddZy)StarRocksSettingsaStarRocks client configuration. Attribute: StarRocks_host (str) : An URL to connect to MyScale backend. Defaults to 'localhost'. StarRocks_port (int) : URL port to connect with HTTP. Defaults to 8443. username (str) : Username to login. Defaults to None. password (str) : Password to login. Defaults to None. database (str) : Database name to find the table. Defaults to 'default'. table (str) : Table name to operate on. Defaults to 'vector_table'. column_map (Dict) : Column type map to project column name onto langchain semantics. Must have keys: `text`, `id`, `vector`, must be same size to number of columns. For example: .. code-block:: python { 'id': 'text_id', 'embedding': 'text_embedding', 'document': 'text_plain', 'metadata': 'metadata_dictionary_in_json', } Defaults to identity map. localhoststrhostiF#intportrootusernamepasswordiddocument embeddingmetadata)rJrKrLrMzDict[str, str] column_mapdefaultdatabase langchaintablect||SN)getattr)selfitems r __getitem__zStarRocksSettings.__getitem__stT""rz.envutf-8 starrocks_ignore)env_fileenv_file_encoding env_prefixextraN)rWrBreturnr)r;r<r=__doc__rCr>rErGrIrNrPrRrXr model_configrrrr@r@Ts6D#D#HcHc  "JHcE3#&! Lrr@ceZdZdZ d dfd ZddZeddZddZddZ d ddZ e d ddZ dd Z d dd Z d dd Z d d d Z d d!d Zd"dZeddZ d# d$dZ d% d&dZxZS)' StarRocksa`StarRocks` vector store. You need a `pymysql` python package, and a valid account to connect to StarRocks. Right now StarRocks has only implemented `cosine_similarity` function to compute distance between two vectors. And there is no vector inside right now, so we have to iterate all vectors and compute spatial distance. For more information, please visit [StarRocks official site](https://www.starrocks.io/) [StarRocks github](https://github.com/StarRocks/starrocks) c h ddl} ddlm}||_t||||_nt|_|j sJ|j jr|j jsJ|j jr,|j jr|j jsJdD]}||j jvrJt|jd}d|j jd |j jd |j jd d |j jd d |j jdd|j jdd |_||_d|_d|_||_d|_t+|j |j,d|j j|j j|j j.|j j0|j jd||_t+|jt5|j2|jy#t$r tdwxYw#t$r d|_YwxYw)zStarRocks Wrapper to LangChain embedding_function (Embeddings): config (StarRocksSettings): Configuration to StarRocks Client rNzVCould not import pymysql python package. Please install it with `pip install pymysql`.)tqdmc|SrTr)xkwargss rz$StarRocks.__init__..sQr)rJrLrKrMtestzCREATE TABLE IF NOT EXISTS .z ( rJz string, rKrLz array, rMzf string ) ENGINE = OLAP PRIMARY KEY(id) DISTRIBUTED BY HASH(id) PROPERTIES ("replication_num" = "1")\)rm'DESC)rCrEuserrIrPr)pymysql ImportErrorrfpgbarsuper__init__configr@rCrErNrPrRlen embed_queryschemadimBS must_escapeembedding_function dist_orderr connectrGrIr)r2) rVrLrvrirqrfr1rz __class__s rruzStarRocks.__init__sb    / !DJ    DK+-DK{{{{{DKK$4$444{{%%$++*>*>4;;CTCTTT< /A ... .. /)''/0 KK0014;;3D3D2EF [[D!"# [[J'() [[K()* [[J'()  &"+ T[[!*'// !!!!%%[[))[[))     T[[!$++6k @   /.DJ /sJ JJJ10J1c8djfd|DS)NrHc3^K|]$}|jvrj|n|&ywrT)r|r{).0crVs r z'StarRocks.escape_str..s/V1!t/?/?*?$''1#QFVs*-)join)rVr-s` r escape_strzStarRocks.escape_strswwVPUVVVrc|jSrT)r}rVs rr6zStarRocks.embeddingss&&&rc dj|}t|j|jjd}g}|D]s}djt |Dcgc]6\}}||k7rd|j t|dn dt|8c}}}|jd|dud|jjd|jjd|d dj|d } | Scc}}w) N,rLrnz array()z1 INSERT INTO rlz)) VALUES z ) rtupleindexrvrNr&rrBr'rPrR) rVtransac column_namesksembed_tuple_index_datanr/_ni_strs r_build_insert_sqlzStarRocks._build_insert_sqls! XXl #!,/55 KK " "; /  #A&/q\  "b"33DOOCG45Q7+CG956 A LL1QCq " #[[))*!DKK,=,=+>atD%!"   !s#;D cj|j||}t|t|j|yrT)rr r2r))rVrr _insert_querys r_insertzStarRocks._inserts*..w E ]#-8rc |xs6|Dcgc]*}t|jdj,c}}|jj}g}|d||d||d|j j t|i} |xs|D cgc]} ic} }ttj|| |d<tt|t| z dk\sJt| j\} } d}|jt| dt| D]} t| | j!|jjd|j"k(sJ|j%| t||k(sf|r|j't)|j*|| g }|j-g}t|dkDr$|r|j'|j+|| |Dcgc]}|c}Scc}wcc} wcc}w#t.$r:}t0j3d t5|d t7|d gcYd}~Sd}~wwxYw)aInsert more texts through the embeddings and add to the VectorStore. Args: texts: Iterable of strings to add to the VectorStore. ids: Optional list of ids to associate with the texts. batch_size: Batch size of insertion metadata: Optional column data to be inserted Returns: List of ids from adding the texts into the VectorStore. rYrJrKrLrMrNzInserting data...)desctotal)targetr   )rencode hexdigestrvrNr}embed_documentslistmapjsondumpsrwsetzipitemsrsrrzr'rrrstart ExceptionloggererrortyperB)rVtextsr9 batch_sizer5ritcolmap_rr_keysvaluesvies r add_textszStarRocks.add_textss*I5Iad188G,-779I++(( DM3 J  K $"9"9"I"I$u+"V 4e!4"!4 ,/ I,F WZ()3w<#l"334999L..01 f AZZV #6c)n  !$**T[[%;%;K%HIJKtxxWWq!w<:-dll'4IAGGI G !7|aFFH Wd+"#!A# #?J"5.$  LL?47)3CCF87S TI sC/H H 9B HA5H9 HHH I/I IIc D|||fi|}|j|||||S)aCreate StarRocks wrapper with existing texts Args: embedding_function (Embeddings): Function to extract text embedding texts (Iterable[str]): List or tuple of strings to be added config (StarRocksSettings, Optional): StarRocks configuration text_ids (Optional[Iterable], optional): IDs for the texts. Defaults to None. batch_size (int, optional): Batchsize when transmitting data to StarRocks. Defaults to 32. metadata (List[dict], optional): metadata to texts. Defaults to None. Returns: StarRocks Index )r5rr9)r) clsrrLr9rvtext_idsrrictxs r from_textszStarRocks.from_texts<s.2)V.v. ejI V rcd|jjd|jjd}||jjd|jjdz }|d|jj dz }d}d }|d ||zd zzd zz }gd }|d|ddd|d dz }|d|dddz }|d ||zd zzd zz }d|jjd|jj}t |t|j|}|D]"}|d|ddd|ddz }|d|dddz }$|d ||zd zzd zz }|S)zText representation for StarRocks Vector Store, prints backends, username and schemas. Easy to use with `str(StarRocks())` Returns: repr: string to show connection info and data schema z rlz @ :z zusername: z Table Schema: - )namerkeyz|r24sz |z| zDESC FieldTypeKey) rvrPrRrCrErGr r2r))rV_reprwidthfieldsr+q_strrsr.s r__repr__zStarRocks.__repr__Ys"$++"6"6!7q9J9J8K3O DKK$$%Qt{{'7'7&8 DD $T[[%9%9$::TUU *+d22) 9WQZ,,> *+d22 ,,-Qt{{/@/@.ABU doou 5 AA y7C 00@63P PE '%~Z@ @E A *+d22 rcdjtt|}|rd|}nd}d|jjdd|jjdd|d |jjd d |jjd d |jj d |jj d|d|jd|d}t||S)NrzWHERE rHz? SELECT id as id, rKz as document, rMzC as metadata, cosine_similarity_norm(array[z], rLz) as dist, z as embedding FROM rlz z ORDER BY dist z LIMIT ) rrrBrvrNrPrRr~r )rVq_embtopk where_str q_emb_strrs r_build_query_sqlzStarRocks._build_query_sqlssHHSe_-   ,II'' 345'' 34555>K@'' 456'' 456++&&'q):):(;< K??+,&   U rc ^|j|jj|||fi|S)aPerform a similarity search with StarRocks Args: query (str): query string k (int, optional): Top K neighbors to retrieve. Defaults to 4. where_str (Optional[str], optional): where condition string. Defaults to None. NOTE: Please do not let end-user to fill this and always be aware of SQL injection. When dealing with metadatas, remember to use `{self.metadata_column}.attribute` instead of `attribute` alone. The default name for it is `metadata`. Returns: List[Document]: List of Documents )similarity_search_by_vectorr}rx)rVr*r1rris rsimilarity_searchzStarRocks.similarity_searchs9&0t//  # # / / 69 HN  rc |j|||} t|j|}|Dcgc]V}t||jj dt j||jj dXc}Scc}w#t$r:}tjdt|dt|dgcYd}~Sd}~wwxYw)aPerform a similarity search with StarRocks by vectors Args: query (str): query string k (int, optional): Top K neighbors to retrieve. Defaults to 4. where_str (Optional[str], optional): where condition string. Defaults to None. NOTE: Please do not let end-user to fill this and always be aware of SQL injection. When dealing with metadatas, remember to use `{self.metadata_column}.attribute` instead of `attribute` alone. The default name for it is `metadata`. Returns: List[Document]: List of (Document, similarity) rKrM page_contentrMrrrN) rr2r)rrvrNrloadsrrrrrB) rVrLr1rrirq_rr.rs rrz%StarRocks.similarity_search_by_vectors.%%iI> "4??E:C    !"4;;#9#9*#E!F!ZZ$++*@*@*L(MN   LL?47)3CCF87S TI s/BAB  B B C/C CCc |j|jj|||} t|j|Dcgc][}t ||j jdtj||j jd|df]c}Scc}w#t$r:}tjdt|dt|dgcYd}~Sd}~wwxYw) aPerform a similarity search with StarRocks Args: query (str): query string k (int, optional): Top K neighbors to retrieve. Defaults to 4. where_str (Optional[str], optional): where condition string. Defaults to None. NOTE: Please do not let end-user to fill this and always be aware of SQL injection. When dealing with metadatas, remember to use `{self.metadata_column}.attribute` instead of `attribute` alone. The default name for it is `metadata`. Returns: List[Document]: List of documents rKrMrdistrrrN)rr}rxr2r)rrvrNrrrrrrrB)rVr*r1rrirr.rs r'similarity_search_with_relevance_scoresz1StarRocks.similarity_search_with_relevance_scoress&%%  # # / / 69  *$//5A  %&t{{'='=j'I%J!%Adkk.D.DZ.P,Q!RfI      LL?47)3CCF87S TI s0B.A B)&B.)B.. C17/C,&C1,C1ct|jd|jjd|jjy)z, Helper function: Drop data zDROP TABLE IF EXISTS rlN)r2r)rvrPrRrs rdropzStarRocks.drops9  OO#DKK$8$8#94;;;L;L:M N rc4|jjdS)NrM)rvrNrs rmetadata_columnzStarRocks.metadata_columns{{%%j11rc |j||d}t|j|}t|Dcgc]}|d c}|Dcgc]1}t j ||j jd3c}|Dcgc]}||j jd c}|Dcgc]1}t j ||j jd3c}|Dcgc]}|d c}} ttj|tj| d|| } t| } t| D cgc] \} }| | vs |} } }| Scc}wcc}wcc}wcc}wcc}wcc}} w) NrJrLrKrMr)r5r6r8r9r:)dtyper6)r1 lambda_mult)rr2r)r4rrrvrNrnparrayfloat32_results_to_docsr&)rVrLr1fetch_krrirrr.results mmr_selected candidatesrselected_resultss r'max_marginal_relevance_search_by_vectorz1StarRocks.max_marginal_relevance_search_by_vectorsU%%i$?t6"%&Q4&LOGH 1T[[33K@ABGJJq// ;<JRUVQtzz!DKK$:$::$F"GHV*-.Qqy. 2 HHYbjj 1 L !#  &g. *3J*?U$!Q1 CTAUU''KV.Vs) E6E# #E( *6E-& E2 E7E7c |j td|jj|}|j||||||S)NzBFor MMR search, you must specify an embedding function oncreation.)rfilterwhere_document)r6 ValueErrorrxr) rVr*r1rrrrrirLs rmax_marginal_relevance_searchz'StarRocks.max_marginal_relevance_searchs^ ?? "T OO//6 ;;  #) <  rrT)rLrrvOptional[StarRocksSettings]rirr`None)r-rBr`rB)r`r)rrr Iterable[str]r`rB)rrrrr`r)N N) rrr9zOptional[List[dict]]rrDr5Optional[Iterable[str]]rirr` List[str])NNNr)rrrLrr9zOptional[List[Dict[Any, Any]]]rvrrrrrDrirr`rd)r`rB)r List[float]rrDr Optional[str]r`rB)N) r*rBr1rDrrrirr`r7) rLrr1rDrrrirr`r7) r*rBr1rDrrrirr`List[Tuple[Document, float]])r`r)r?) rLz list[float]r1rDrrDrfloatrirr`zlist[Document])rrNN)r*rBr1rDrrDrr rOptional[Dict[str, str]]rr rirr`r7)r;r<r=rarurpropertyr6rrr classmethodrrrrrrrrrr __classcell__)rs@rrdrds "/3B7B7,B7 B7  B7HW''49+/'+ 77(7 7 % 7  7 7r 59.2,02  ,  *  86IM (+8E 6BF    1> QT  4#' ## #! #  #  #LBF## #1>#QT# %#J 22              F +/37        )  1    rrdcJt|Dcgc]\}}| c}}Scc}}wrT)_results_to_docs_and_scores)rdocrs rrr5s9'B CFCC CC Cs ct|d|d|dDcgc]}t|d|dxsi|df c}Scc}w)Nr8r9r:rrrr)rr)rr,s rrr9s] K K K     vay6!9? CVAYO  s#?)rrBrrr`bool)rrr`r)r)rr*rBr`zList[dict[str, Any]])rrr`r7)rrr`r)0 __future__rrlogginghashlibr threadingrtypingrrrr r r r r numpyrlangchain_core.documentsrlangchain_core.embeddingsrlangchain_core.vectorstoresrpydantic_settingsrrtyping_extensionsr&langchain_community.vectorstores.utilsr getLoggerrrrr r2rBrDr rMetadatar4r@rdrrrrrr"s" MMM-03>'M       2 3c3t344 5%)%3 3lh h V Dr