ihIHPdZddlmZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZmZddlmZmZddlmZdd lmZdd lmZdd lmZdd lmZej:eZd gZ eGddZ!ddhZ"hdZ#ddZ$GddeZ%GddeZ&eGddeZ'y)zLoads YouTube transcript.) annotationsN)Enum)Path)AnyDict GeneratorListOptionalSequenceUnion)parse_qsurlparse) ParseError)Document)model_validator) dataclass) BaseLoaderz0https://www.googleapis.com/auth/youtube.readonlyceZdZUdZej dz dz Zded<ej dz dz Zded<ej dz dz Z ded<dd Z e d e dd Z dd Zy)GoogleApiClientaGeneric Google API Client. To use, you should have the ``google_auth_oauthlib,youtube_transcript_api,google`` python package installed. As the google api expects credentials you need to set up a google account and register your Service. "https://developers.google.com/docs/api/quickstart/python" *Security Note*: Note that parsing of the transcripts relies on the standard xml library but the input is viewed as trusted in this case. Example: .. code-block:: python from langchain_community.document_loaders import GoogleApiClient google_api_client = GoogleApiClient( service_account_path=Path("path_to_your_sec_file.json") ) z .credentialszcredentials.jsonrcredentials_pathservice_account_pathz token.json token_pathc.|j|_yN)_load_credentialscredsselfs j/var/www/html/dev/engine/venv/lib/python3.12/site-packages/langchain_community/document_loaders/youtube.py __post_init__zGoogleApiClient.__post_init__2s++- beforemodec|jjds&|jjds td|jS)DValidate that either folder_id or document_ids is set, but not both.rr-Must specify either channel_name or video_idskwargsget ValueErrorclsvaluess r#validate_channel_or_videoIds_is_setz3GoogleApiClient.validate_channel_or_videoIds_is_set5sC }}  !34V]]=N=N "> LM M}}r!c ddlm}ddlm}ddlm}ddlm}ddlm }d}|jjr.|j jt|jS|jjr)|j!t|jt"}|r |j$s|r/|j&r#|j(r|j+|n;|j-t|j.t"}|j1d }t3|jd 5}|j5|j7ddd|S|S#t$r tdwxYw#1swY|SxYw) zLoad credentials.r)Request)service_account) Credentials)InstalledAppFlowYouTubeTranscriptApiYou must run`pip install --upgrade google-api-python-client google-auth-httplib2 google-auth-oauthlib youtube-transcript-api` to use the Google Drive loaderN)portw)google.auth.transport.requestsr1 google.oauth2r2google.oauth2.credentialsr3google_auth_oauthlib.flowr4youtube_transcript_apir6 ImportErrorrexistsfrom_service_account_filestrrfrom_authorized_user_fileSCOPESvalidexpired refresh_tokenrefreshfrom_client_secrets_filerrun_local_serveropenwriteto_json) rr1r2r3r4r6rflowtokens rrz!GoogleApiClient._load_credentials@sH  > 5 = B C  $ $ + + -"..HHD--.  ?? ! ! #99#doo:NPVWEEKK5+>+> gi('@@--.--1-5doos+ -u EMMO, - u ; 1  4 - sE)= F)E>F NreturnNoner.rrQr)rQr)__name__ __module__ __qualname____doc__rhomer__annotations__rrr r classmethodr/rr!rrrs*'TYY[>9vid.plusyoutu.be youtube.com m.youtube.comwww.youtube.comwww.youtube-nocookie.comct|}|jtvry|jtvry|j }|j dr9|j}t|}d|vr|d}t|tr|n|d}n0y|j jd}|jdd}t|dk7ry|S)zEParse a YouTube URL and return the video ID if valid, otherwise None.Nz/watchvr/ )rschemeALLOWED_SCHEMESnetlocALLOWED_NETLOCSpathendswithqueryr isinstancerBlstripsplitlen)url parsed_urlrmro parsed_queryidsvideo_ids r_parse_video_idryts#J// ??D }}X   , s#C(c2sAH%%c*::c?2& 8} Or!ceZdZdZdZdZdZy)TranscriptFormatz3Output formats of transcripts from `YoutubeLoader`.textlineschunksN)rTrUrVrWTEXTLINESCHUNKSr[r!rr{r{s= D E Fr!r{ceZdZdZdddej ddf d dZeddZe ddZ dd Z dd Z dd Z dd Zy) YoutubeLoaderz!Load `YouTube` video transcripts.FenNxc||_d|i|_||_||_t |t r |g|_n||_||_||_||_||_ y)z!Initialize with YouTube video ID.sourceN) rx _metadataadd_video_infolanguagerprB translationtranscript_formatcontinue_on_failurechunk_size_seconds)rrxrrrrrrs r__init__zYoutubeLoader.__init__s`! "H-,  h $%JDM$DM&!2#6 "4r!c>t|}|std|d|S)z*Extract video ID from common YouTube URLs.z.Could not determine the video ID for the URL "z".)ryr+) youtube_urlrxs rextract_video_idzYoutubeLoader.extract_video_ids1#;/@ RP r!c 6|j|}||fi|S)z|Given a YouTube URL, construct a loader. See `YoutubeLoader()` constructor for a list of keyword arguments. )r)r-rr)rxs rfrom_youtube_urlzYoutubeLoader.from_youtube_urls$ '' 48&v&&r!c t|d\}}t|d\}}tdjtd|i|j||dd|dd|dd|j d|dd  S) z0Create Document from chunk of transcript pieces.< c*|djdSNr|rstrip) chunk_pieces rz4YoutubeLoader._make_chunk_document..s F(;(A(A#(Fr!02d: https://www.youtube.com/watch?v=z&t=s) start_secondsstart_timestampr page_contentmetadata)divmodrjoinmaprrx)r chunk_pieceschunk_start_secondsmrhs r_make_chunk_documentz"YoutubeLoader._make_chunk_documents)2.1a}1F U..!4&'WAaWAaW#=34==/)*!-   r!c#Kg}d}|j}|D]L}|d|dz}||kDr)|r|j||g}|}||jz }|j|Nt|dkDr|j||yyw)Nrstartduration)rrappendrs)rtranscript_piecesrrchunk_time_limittranscript_piece piece_ends r_get_transcript_chunksz$YoutubeLoader._get_transcript_chunkss.0 22 1 2 (14DZ4PPI++33LBUVV! &6# D$;$;;    0 1 2 | q ++L:MN N !sBBcv ddlm}m}m}|j r+|j }|jj| |j|j} |j|j}|j|j|j}|j}|j t"j$k(r4dj't)d|}t+||jgS|j t"j,k(rt/t)d|S|j t"j0k(rt/|j3|St5d #t$r t dwxYw#|$rgcYSwxYw#|$r|jdg}Y.s-=f-E-K-KC-Pr!rc t|djdttd|j S)Nr|rc|ddk7S)Nrr|r[)items rrz6YoutubeLoader.load...."sT!W->r!r)rrdictfilteritemsrs rrz$YoutubeLoader.load..s=X%5f%=%C%CC%H!%" >@P@V@V@X".r!zUnknown transcript format.)r>rrr6r?r_get_video_inforupdatelist_transcriptsrxfind_transcriptrr translatefetchrr{rrrrrlistrrr+)rrrr6 video_infotranscript_list transcriptrs rloadzYoutubeLoader.loads      --/J NN ! !* - 2CCDMMRO A(88GJ    '#--d.>.>?J2<2B2B2D  ! !%5%:%: :P%J *t~~NO O  # #'7'='= =&    # #'7'>'> >334EFG G9: :k O  # I  ! A(88$@J As/ E8F F8F FFF87F8cx ddlm}|d|j}|jxsd|j xsd|j xsd|jxsd|jr|jjdnd|jxsd|jxsdd}|S#t$r tdwxYw)zGet important video information. Components include: - title - description - thumbnail URL, - publish_date - channel author - and more. r)YouTubezVCould not import "pytube" Python package. Please install it with `pip install pytube`.rUnknownz%Y-%m-%d %H:%M:%S)title description view_count thumbnail_url publish_datelengthauthor) pytuberr?rxrrviewsrrstrftimerr)rrytrs rrzYoutubeLoader._get_video_info/s  &7 G HXX*>>6Y((-a--:OO445HIiin1ii,9  # ?  s B$$B9)rxrBrboolrzUnion[str, Sequence[str]]r Optional[str]rr{rrrint)rrBrQrB)rrBr)rrQr)r List[Dict]rrrQr)rrrQzGenerator[Document, None, None]rQList[Document])rQr)rTrUrVrWr{rr staticmethodrrZrrrrrr[r!rrrs+ %.2%).>.C.C$)"%555, 5 # 5 , 5"5 50'' & =@  *O!+O (O(=;~r!rceZdZUdZded<dZded<dZded<d Zd ed <d Zd ed<dZ d ed<ddZ ddZ e de ddZddZd dZd!dZd"dZd#dZd$dZy)%GoogleApiYoutubeLoaderaLoad all Videos from a `YouTube` Channel. To use, you should have the ``googleapiclient,youtube_transcript_api`` python package installed. As the service needs a google_api_client, you first have to initialize the GoogleApiClient. Additionally you have to either provide a channel name or a list of videoids "https://developers.google.com/docs/api/quickstart/python" Example: .. code-block:: python from langchain_community.document_loaders import GoogleApiClient from langchain_community.document_loaders import GoogleApiYoutubeLoader google_api_client = GoogleApiClient( service_account_path=Path("path_to_your_sec_file.json") ) loader = GoogleApiYoutubeLoader( google_api_client=google_api_client, channel_name = "CodeAesthetic" ) load.load() rgoogle_api_clientNr channel_namezOptional[List[str]] video_idsTrrrrBcaptions_languageFrcX|j|jj|_yr)_build_youtube_clientrryoutube_clientrs rr z$GoogleApiYoutubeLoader.__post_init__vs!"889O9O9U9UVr!cb ddlm}ddlm}|dd|S#t$r t dwxYw)Nr)buildr5r7youtubev3) credentials)googleapiclient.discoveryrr>r6r?)rrrr6s rrz,GoogleApiYoutubeLoader._build_youtube_clientysA  7 CY%88 1  s .r"r#c|jjds&|jjds td|jS)r&rrr'r(r,s rr/z:GoogleApiYoutubeLoader.validate_channel_or_videoIds_is_sets=}}  09J9J;9WLM M}}r!cTddlm}m}|j|} |j |j g}j}dj|Dcgc]}|djdc}S#|$r%|D]}|j |j }YjwxYwcc}w)Nr)rr6rr|) r>rr6rrrrrrr) rrxrr6rravailable_transcriptrts r_get_transcripe_for_video_idz3GoogleApiYoutubeLoader._get_transcripe_for_video_idsR.??I (88$:P:P9QRJ ',,.xx7HI!6-IJJ ! (7 $1;;D    & & ( T!WY !#''03  r!c|jjjd|dd}|j}|dddd}|S)Nrchannel)rqtype maxResultsrr channelId)rsearchrr)rrrequestresponse channel_ids r_get_channel_idz&GoogleApiYoutubeLoader._get_channel_ids_%%,,.33 4  ??$g&q)$/ < r!c|jjjd|}|j}|dddddS)NcontentDetailsrrrrelatedPlaylistsuploads)rchannelsrr)rr r r s r_get_uploads_playlist_idz/GoogleApiYoutubeLoader._get_uploads_playlist_ids[%%..055!6 ??$ #$456HI)TTr!c  ddlm}m}|j |}|j |}|j jjd|d}g}||j} | dD]s} | dd d } d | i} |jr(| djd | j| d |j| } |jt| | u|j j)j+|| }||S#t$r tdwxYw#||t f$r>}|j"r%t$j'd d| dd d|zn|Yd}~d}~wwxYw)Nr)rrzTYou must run`pip install --upgrade youtube-transcript-api` to use the youtube loaderr2)r playlistIdrrsnippet resourceIdvideoId thumbnailsrzError fetching transscript rrz , exception: )r>rrr?r rr playlistItemsrrrpoprrrrrrloggererrorr list_next)rrr)rrr uploads_playlist_idr rr rrx meta_datares r_get_document_for_channelz0GoogleApiYoutubeLoader._get_document_for_channels  ))'2 ";;JG%%335::*;   !(H!)  ?<8C&1 &&O'' 5$$T)_5#'#D#DX#NL$$ )5%. .))002< K// 9 !$t*Y"7!8 aSIJ  s#D8-D.D+. E8:3E33E8c"g}|jr,|j|j|j|S|jr:|j|jDcgc]}|j |c}|St dcc}w)zLoad documents.r')rextendr"rrr+)r document_listrxs rrzGoogleApiYoutubeLoader.loads     !?!?@Q@Q!R S^^  %)NN 33H= LM M s B rP)rrrQrrS)rxrBrQrB)rxrBr)rrQr)rrBrQrB)r rBrQrB)rrBr)rrQrr)rTrUrVrWrYrrrrrr rrrZr/rrr rr"rr[r!rrrQs8'&"&L-&%)I")ND!s! %%W9 (#$ K   U3jr!r)rtrBrQr)(rW __future__rloggingenumrpathlibrtypingrrrr r r r urllib.parser rxml.etree.ElementTreerlangchain_core.documentsrpydanticrpydantic.dataclassesr)langchain_community.document_loaders.baser getLoggerrTrrDrrjrlryr{rrr[r!rr2s"HHH+,-$*@   8 $ < = NN Nb7#<ttJtn rZr rr!