ihX ddlmZddlZddlmZddlmZmZm Z m Z m Z ddl m Z ddlmZddlmZmZddlmZdd lmZe eefZed ZGd d eZy) ) annotationsN)Path)IteratorLiteralOptionalSequenceUnion)Document)BaseBlobParser) BlobLoaderFileSystemBlobLoader) GenericLoader) get_parserdefaultc eZdZdZ d d fd Z ddZeddddd ddd  dd ZxZS)ConcurrentLoaderz%Load and pars Documents concurrently.c4t|||||_y)N)super__init__ num_workers)self blob_loader blob_parserr __class__s m/var/www/html/dev/engine/venv/lib/python3.12/site-packages/langchain_community/document_loaders/concurrent.pyrzConcurrentLoader.__init__s k2&c#Ktjj|j5}|jj Dchc](}|j |jj|*}}tjj|D]}|jEd{ dddycc}w7#1swYyxYww)z.Load documents lazily with concurrent parsing.) max_workersN) concurrentfuturesThreadPoolExecutorrr yield_blobssubmitr lazy_parse as_completedresult)rexecutorblobr!futures r lazy_loadzConcurrentLoader.lazy_load"s   2 2((3  + !,,88: 0 0 ; ;TBG%,,99'B +!==?** + + + + + +s:+CC -B?77C.C/C6 C?CC Cz**/[!.]*NFr)globexcludesuffixes show_progressparserr parser_kwargsct|||||} t|trE|dk(r4|jtjk7r|jdi|xsi} nt|} n|} || | |S)aCreate a concurrent generic document loader using a filesystem blob loader. Args: path: The path to the directory to load documents from. glob: The glob pattern to use to find documents. suffixes: The suffixes to use to filter documents. If None, all files matching the glob will be loaded. exclude: A list of patterns to exclude from the loader. show_progress: Whether to show a progress bar or not (requires tqdm). Proxies to the file system loader. parser: A blob parser which knows how to parse blobs into documents num_workers: Max number of concurrent workers to use. parser_kwargs: Keyword arguments to pass to the parser. )r-r.r/r0r)rr,)r isinstancestrrr) clspathr-r.r/r0r1rr2rrs rfrom_filesystemz ConcurrentLoader.from_filesystem0sx6+ '   fc ""s~~9Q9Q'Q,cnnE 0CE (0  K; EEr)r)rr rr rintreturnNone)r:zIterator[Document])r7 _PathLiker-r5r.z Sequence[str]r/zOptional[Sequence[str]]r0boolr1zUnion[DEFAULT, BaseBlobParser]rr9r2zOptional[dict]r:r) __name__ __module__ __qualname____doc__rr+ classmethodr8 __classcell__)rs@rrrs/  ''$' '  ' +  + !#,0#1:(,)F)F )F  )F * )F)F/)F)F&)F )F)Frr) __future__rconcurrent.futuresr pathlibrtypingrrrrr langchain_core.documentsr )langchain_community.document_loaders.baser 1langchain_community.document_loaders.blob_loadersr r ,langchain_community.document_loaders.genericr5langchain_community.document_loaders.parsers.registryrr5r<DEFAULTrr,rrrNsP"??-DGL #t)  ) DF}DFr