7|hcddlmZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z m Z mZmZmZddlmZddlmZddlmZdd lmZdd lmZmZdd lmZdd lmZej@e!Z"d Z#dZ$dZ%dZ&gdZ'gdZ(dgZ)gdZ*e'e(e)e*dZ+Gdde,eZ-GddeZ.GddeZ/GddeZ0GddeZ1Gd d!eZ2d-d"Z3d.d#Z4d/d$Z5d0d%Z6d1d&Z7 d2 d3d'Z8d4d(Z9d5d)Z:d6d*Z;Gd+d,eZr>QseD#, I# I)B 4M* GOM(GS4r5r>c(eZdZUdZded< ded<y) FrameworkzPebblo Framework instance.r9nameversionNr;r4r5r6rMrMjs$ I L#r5rMcpeZdZUdZded< ded< ded< ded< ded < d ed < ded < d ed <y)AppzPebblo AI application.r9rNownerrD descriptionload_idr>rKrM frameworkplugin_versionclient_versionNr;r4r5r6rQrQssJ I J! L- %'**r5rQceZdZUdZded< ded< ded< ded< ded< d ed < d ed < ded < ded< d ed<y)DoczPebblo document.r9rNrRlistdocsrVrTdictloader_detailsbool loading_end source_ownerclassifier_locationanonymize_snippetsNr;r4r5r6rYrYs[ I0 J J. L-+7,%Ur5rYc|rd|vs d|dk(s|dvr|Stj|}|jr|j}t |S)zReturn an absolute local path for a local file/directory, for a network related path, return as is. Args: path (str): Relative path to be resolved. Returns: str: Resolved absolute path. z:///r)unknown-r+)pathlibPathexistsresolver9)rB full_paths r6 get_full_pathrlsY  TM 47N 1 1  T"I%%' y>r5cLtjD] \}}||vs |cSy)zReturn loader type among, file, dir or in-memory. Args: loader (str): Name of the loader, whose type is to be resolved. Returns: str: One of the loader type among, file/dir/in-memory. unsupported)LOADER_TYPE_MAPPINGitems)loader loader_typeloaderss r6get_loader_typerts4!4 9 9 ; W W   r5cVddlm}m}m}m}d}t |t stjd|S|j} d|vrUt ||rd|jd|j}nt ||rd|jd|j}nbd |vr|d }|rVd |vrQ|d }|rI|d|}n@d |vr|d }n5d |vr|d }n*d |vr-|d }|rt |tr t|dkDr|d}nt ||rd}nt ||rd|j}n|j j"dk(r|j%dr|j%d} d| }n|j%dr6|j%dg} dj'| D cgc]} d| d c} }nF|j%dr5|j%dg} dj'| D cgc]} d| d c} }t+t-|Scc} wcc} w#t($rY)wxYw)zReturn an absolute source path of source of loader based on the keys present in Document. Args: loader (BaseLoader): Langchain document loader, derived from Baseloader. r)r%rr&rrfzGloader is not derived from BaseLoader, source location will be unknown!bucketzgc://rdzs3://sourcechannelrB file_path web_pathsr+z notiondb://r' folder_idz+https://drive.google.com/drive/u/2/folders/file_idsz, z https://drive.google.com/file/d/z/view document_idsz#https://docs.google.com/document/d/z/edit)$langchain_community.document_loadersr%rr&r isinstancerloggererror__dict__rvblobkeyrZlen database_id __class__r.getjoin Exceptionrlr9)rqr%rr&rlocation loader_dictrxrzr{r|file_idr}doc_ids r6get_loader_full_pathrsNH fj ) U //K/ { "&-0"6==/6;;-@FL1"6==/6::,?  $"8,HI4%i0"*1WI6H { ""6*H K '";/H K '#K0IZ 48S^a=O$Q<  0"H  /$V%7%7$89H    & &*= ={+'OOK8 H T,&??:r:99(0#;7)5I 0*~rB 99'3">fXUK X ''!    s1E H# H0:H* H7H H H('H(c  t}td|jdd}tj}t |j tjd|jdd|j|jt|jdd|jd d }d |jvrd |_ d |_ tjd|tjd|||fS)zFetch the current Framework and Runtime details. Returns: Tuple[Framework, Runtime]: Framework and Runtime for the current app instance. langchainlibrary_versionN)rNrOPWDrFrerKruntime_version)rArBrFrGrHrErIrJDarwindesktopzMac OSXz framework zruntime )r rMrrFunamer>noderGenvironsystemrOget_ipr@rKrdebug) runtime_envrUrrKs r6 get_runtimers *+K +//2CT"JI NN E ZZ ZZ Y7 <<== 8I6$):IF G7::  # LL:i[)* LL8G9%& g r5cddl}|j} |j|}|S#t$r|jd}Y|SwxYw)zJFetch local runtime ip address. Returns: str: IP address rN localhost)socket gethostname gethostbynamer)rrA public_ips r6rr.sY     D6((.   6((5 6s)AAc$g}g}d}|D]q}t|jjd}||kDr|j|g?||z|kDr|j|g}d}|j|||z }s|r|j||S)a Generate batches of documents based on page_content size. Args: docs: List of documents to be batched. max_batch_size: Maximum size of each batch in bytes. Defaults to 100*1024(100KB) Returns: List[List[Document]]: List of batches of documents rutf-8)r page_contentencodeappend)r[max_batch_sizebatches current_batchcurrent_batch_sizedocdoc_sizes r6generate_size_based_batchesr>s%'G$&M+C,,33G<= n $ NNC5 !!H,~=}- " %&"   % ( * !+&}% Nr5c ddl}tj|j}|j |j }|S#t $rd}Y|SwxYw)zFetch owner of local file path. Args: file_path (str): Local file path. Returns: str: Name of owner. rNre)pwdrGstatst_uidgetpwuidpw_namer)ryrfile_owner_uidfile_owner_names r6get_file_owner_from_pathrfsV$+22,,~6>>  $# $s>A AAc|syd}tjj|r!tjj|}|Stjj |rd}tj |D]o\}}}|D]d}tjj ||}tjj|rC|tjj|z }fq|}|S)zFetch size of source path. Source can be a directory or a file. Args: source_path (str): Local path of data source. Returns: int: Source size in bytes. r)rGrBisfilegetsizeisdirwalkrislink) source_pathsize total_sizedirpath_ filenamesffps r6get_source_sizerys  D ww~~k"ww{+ K { # %'WW[%9 6 !GQ  6WW\\'1-ww~~b)"''//""55J 6 6  Kr5c>|jd}t|}|S)zCalculate the content size in bytes: - Encode the string to bytes using a specific encoding (e.g., UTF-8) - Get the length of the encoded bytes. Args: data (str): Data string. Returns: int: Size of string in bytes. r)rr)dataencoded_contentrs r6calculate_content_sizers!kk'*O  D Kr5c(eZdZUdZded< dZded< ded< ded< d Zd ed < dfd Zdd Z d ddZ ddZ dddZ ddZ e d ddZe ddZe ddZxZS) PebbloLoaderAPIWrapperzWrapper for Pebblo Loader API.rDapi_keyr?r9raclassifier_url cloud_urlFr^rbc t|ddd|d<t|ddt|d<t|ddt|d<t|d i|y) z%Validate that api key in environment.rPEBBLO_API_KEYrCrPEBBLO_CLASSIFIER_URLrPEBBLO_CLOUD_URLNr4)r _DEFAULT_CLASSIFIER_URL_DEFAULT_PEBBLO_CLOUD_URLsuper__init__)selfkwargsrs r6rzPebbloLoaderAPIWrapper.__init__sh0 I/ y$8 $&=?V$  3 K!35N { "6"r5cLd}|jd}|jdk(rL|j}|jtj j }|jd|||}|jr|jd}|rAtj|jjd}|jd|i|jdti|jtj j }|jd|||}yy) z Send app discovery request to Pebblo server & cloud. Args: app (App): App instance to be discovered. NT exclude_unsetr?POST cloud_requestpebblo_server_versionpebblo_client_version)r\ra _make_headersrr-r3value make_requestrjsonloadstextrupdatePLUGIN_VERSIONr) rapp pebblo_resppayloadheadersapp_discover_urlrpebblo_cloud_urlrs r6send_loader_discoverz+PebbloLoaderAPIWrapper.send_loader_discovers (((.  # #w .((*G&&'(B(B(H(H'IJ ++F4DgwWK <<((t( t|||||}tjd||jjt t |jj r|jj ngt |j|jtjk\r$tjd|j|S|jtjk\r$tjd|j|S|jtjk7r"tjd|j|S#t$rtjd|Yyt$r }tjd|Yd}~yd}~wwxYw) a Make a request to the Pebblo API Args: method (str): HTTP method (GET, POST, PUT, DELETE, etc.). url (str): URL for the request. headers (dict): Headers for the request. payload (Optional[dict]): Payload for the request (for POST, PUT, etc.). timeout (int): Timeout for the request in seconds. Returns: Optional[Response]: Response object if the request is successful. )methodurlrrtimeoutz5Request: method %s, url %s, len %s response status %szPebblo Server: Error z$Pebblo received an invalid payload: z-Pebblo returned an unexpected response code: zUnable to reach server %sz'An Exception caught in make_request: %sN)rrrr r9rbody status_coderINTERNAL_SERVER_ERRORr BAD_REQUESTrOKrr)r r rrr responsers r6rz#PebbloLoaderAPIWrapper.make_requestksd* I3gwH LLG  $$C1A1A1F1F((--BOPH(()  ##z'G'GG!6x7K7K6LMNO%%)?)??!Ehmm_UVO %%6C++,. O = NN6 < I NNDa H H Is,CEAEAEF4F<FFc g}d}|Dcgc]}|j}}d}|D]Y}|jdi}|jdg} |ddk(rt|jd|d} n+t|jd |jd|} |jd t| } |jd t | } t |jd } t | }||z }|jd dxsd}|j| | ||jdijd| d| rd| ini| d| ini|ddk(s@|rD|jd|d<d}\||fScc}w)a Prepare documents for classification. Args: docs_with_id (List[IndexedDocument]): List of documents to be classified. source_path (str): Source path of the documents. loader_details (dict): Contains loader info. Returns: Tuple[List[dict], int]: Documents and the aggregate size of the source. rFmetadataauthorized_identitiesrqr(rwrrkrRrrr:N last_modified)rrr:r file_ownersource_path_sizesource_full_urlT)r\rrlrrr9rr)rrr]r[rr doc_contentsource_path_update doc_metadatadoc_authorized_identitiesdoc_source_pathdoc_source_ownerdoc_source_sizerpage_content_sizers r6rz6PebbloLoaderAPIWrapper.prepare_docs_for_classifications$ !-9:csxxz: :". *C77:r2L(4(8(89PRT(U %h'+=="/ $$X~m/LM##0 $$#$((;?# ,//1/B  +..v7WXOsww~67L 6| D  !%6 6 !WWWd+0qF KK'#2#%(WWZ%<%@%@%Q"2 512KL+6,_= (x(,>>*0<0@0@AR0S}-%)"]. *^***c;sFc |D]|}|j|di}|j|jd|jd|jdi|jdid|jd~y) z Update the document data with classified information. Args: docs (List[dict]): List of document data to be updated. classified_docs (dict): The dictionary containing classified documents. r: pb_checksumloader_source_pathentitiestopics)r"r#r$r%rN)rrr)r[rdoc_dataclassified_datas r6rz&PebbloLoaderAPIWrapper.update_doc_datas H-11(72CRHO OO#2#6#6}#E*9*=*=>R*S / 3 3J C-11(B?   LL  r5)rr)rrQreturnNone)F) rList[IndexedDocument]rrQr]r\r_r^r(r\)rr\r(r))rr^r(r\)rrQr[ List[dict]r]r\r`r9rintr_r^r(r\)N) r r9r r9rr\rzOptional[dict]r r,r(zOptional[Response])rr*rr9r]r\r(zTuple[List[dict], int])r[r+rr\r(r))r.r/r0r1r<rarbrrrrrr staticmethodrrr __classcell__)rs@r6rrs( "&&I!!&!$$U #NH" <+<< <  <  <| U.* ** *  * # ** *X #' // // /  /  //bD++D+D+D+ D+D+L  r5r)rBr9r(r9)rqr9r(r9)rqrr(r9)r(zTuple[Framework, Runtime])r(r9)r)r[zList[Document]rr,r(zList[List[Document]])ryr9r(r9)rr9r(r,)rr9r(r,)= __future__rrloggingrGrgrFenumrhttprtypingrrrr r langchain_core.documentsr langchain_core.envr langchain_core.utilsr pydanticrrequestsrrrequests.exceptionsr)langchain_community.document_loaders.baser getLoggerr.rrrrBATCH_SIZE_BYTES file_loader dir_loader in_memory cloud_folderror9r-r8r>rMrQrYrlrtrrrrrrrrr4r5r6rBs="  33-65&0@   8 $11         -S$-%h%5i52$ $+)+*V)V2. E(P> "1;% %*-%%P&2  W YW r5