7|hw#ddlmZddlZddlZddlmZddlmZmZm Z m Z m Z ddl m Z ddlmZer ddlZddlmZddZGd d eZdd ZGd d eZeZy)) annotationsN)Path) TYPE_CHECKINGDictListOptionalUnion)Document) BaseLoader) EntityLikec6|d}|d}|d}|d|d|dS)zBCombine message information in a readable format ready to be used.datefromtextz on z:  )rowrsenderrs l/var/www/html/test/engine/venv/lib/python3.12/site-packages/langchain_community/document_loaders/telegram.pyconcatenate_rowsrs6 v;D [F v;DXT$r$t ,,c eZdZdZddZddZy)TelegramChatFileLoaderzLoad from `Telegram chat` dump.c||_y)zInitialize with a path.N) file_path)selfpaths r__init__zTelegramChatFileLoader.__init__s rct|j}t|d5}tj|}ddddj ddD}dt |i}t||gS#1swY?xYw) Load documents.utf8encodingNc3hK|]*}|ddk(r t|dtr t|,yw)typemessagerN) isinstancestrr).0r's r z.TelegramChatFileLoader.load..'s6 v)+ 76?C0P W % s02messagessource page_contentmetadata)rropenjsonloadjoinr)r )rpfdrr0s rr3zTelegramChatFileLoader.load s   !f %  ! A ww Z=   c!f%dX>??  s A77BN)rzUnion[str, Path]returnList[Document])__name__ __module__ __qualname____doc__rr3rrrrrs)@rrcddlm}|dgdd}t|tr|g}|Dcgc]}t |}}t |D]\}}|dz|j d <g}|D]}|j|j}t |D]a\}} t | |j d |d  }|j d d |j d |j d<|j|c|Scc}w)zIConvert a string or list of strings to a list of Documents with metadata.r)RecursiveCharacterTextSplitteri )r .!?, r$) chunk_size separators chunk_overlap)r/page)rLchunkr.-rMr-) langchain_text_splittersr@r(r)r enumerater0 split_textr/append) rr@ text_splitterrL page_docsidoc doc_chunkschunksrMs r text_to_docsrY1sG2>M $v9=>t,>I>I&%3 1u V%J#))#*:*:;!&) #HAu"cll66JUV-WC), V(<'=Qs||G?T>U%VCLL "   c "  ## %?sC:cbeZdZdZ d d dZd dZd dZ d dZd dZy)TelegramChatApiLoaderz)Load `Telegram` chat json directory dump.NcJ||_||_||_||_||_y)aIInitialize with API parameters. Args: chat_entity: The chat entity to fetch data from. api_id: The API ID. api_hash: The API hash. username: The username. file_path: The file path to save the data to. Defaults to "telegram_data.json". N) chat_entityapi_idapi_hashusernamer)rr]r^r_r`rs rrzTelegramChatApiLoader.__init__Vs($'     "rc Kddlm}g}||j|j|j4d{}|j |j 23d{}|jdu}|r|jjnd}|j|j|j|jj|j||d77}6dddd{7n#1d{7swYnxYwt|j dd5}t#j$||dd dddy#1swYyxYww) z8Fetch data from Telegram API and save it as a JSON file.r)TelegramClientN) sender_idrr message.idis_reply reply_to_idwzutf-8r"F) ensure_asciiindent) telethon.syncrbr`r^r_ iter_messagesr]reply_toreply_to_msg_idrRrcrr isoformatidr1rr2dump)rrbdataclientr'rerfr6s rfetch_data_from_telegramz.TelegramChatApiLoader.fetch_data_from_telegramns0!$--dmmL  PV!'!5!5d6F6F!G  g"++47BJg..>>PT  %,%6%6 ' ' 6 6 8&-jj$,'2    !G     $..# 8 =A IIdAE! < = = =s5E CE C.CCCA8C.E CC. E 'C*(E .D4C7 5D<E D>5 E >EE c dfd ||d}||djdg}|djt|d<|dDcic]}||g||z}}|Scc}w)a Create a dictionary of message threads from the given data. Args: data (pd.DataFrame): A DataFrame containing the conversation data with columns: - message.sender_id - text - date - message.id - is_reply - reply_to_id Returns: dict: A dictionary where the key is the parent message ID and the value is a list of message IDs in ascending order. cr||d|k(dj}g}|D]}||g||zz }|S)a^ Recursively find all replies to a given parent message ID. Args: parent_id (int): The parent message ID. reply_data (pd.DataFrame): A DataFrame containing reply messages. Returns: list: A list of message IDs that are replies to the parent message ID. rfrd)tolist) parent_id reply_datadirect_replies all_repliesreply_id find_repliess rr}z@TelegramChatApiLoader._get_message_threads..find_repliessc( =(AY(NOfh  K* OzL:,NNN  O rrerf)subsetrd)rxintry pd.DataFramer9z List[int])dropnaastyper)rrrparent_messagesreply_messagesrxmessage_threadsr}s @r_get_message_threadsz*TelegramChatApiLoader._get_message_threadss$ 0Z 001d:./66}o6N)7}(E(L(LS(Q}% -\:   {\)^%LL L    s A%c.d}|jD]l\}}||dj|jddj}|Dcgc] }t |}}|dj |dzz }n|j Scc}w)aw Combine the message texts for each parent message ID based on the list of message threads. Args: message_threads (dict): A dictionary where the key is the parent message ID and the value is a list of message IDs in ascending order. data (pd.DataFrame): A DataFrame containing the conversation data: - message.sender_id - text - date - message.id - is_reply - reply_to_id Returns: str: A combined string of message texts sorted by date. r$rdr)byrrFz. )itemsisin sort_valuesrwr)r4strip)rrrr combined_textrx message_ids message_textselems r_combine_message_textsz,TelegramChatApiLoader._combine_message_textss* '6&;&;&= = "I{T,',,[9:'0  4AA4SYAMA SXXm4u< rrtrrr3rrrr[r[Ssy3-1 $"&"&- #)## #  #  #0=.9v$%3$%;G$% $%L",rr[)rrr9r))rzUnion[str, List[str]]r9r:) __future__rrr2pathlibrtypingrrrrr langchain_core.documentsr )langchain_community.document_loaders.baser rrtelethon.hintsr rrrYr[TelegramChatLoaderrrrrsV" ==-@)-@Z@0Du,Ju,r,r