7|hT dZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z mZmZmZmZmZmZmZmZmZddlmZmZddlmZddlmZmZddlm Z m!Z!m"Z"m#Z#m$Z$m%Z%e rddl&Z'ddl(m)cm*cm+Z,ejZe.Z/d Z0d Z1d Z2d&d Z3GddZ4e4Z5e4Z6Gdde7e Z8Gdde!Z9Gdde!Z:GddZ;dee7deddd e7fdZ?dedd ee7ddffdZ@eeAe4fZBe7ZCGd d!ZDeDZEe7ZFGd"d#e9e:eeEeFfZGee7eefZHeAZIGd$d%e9e:eeHeIfZJeGZKeJZLeDZMy)'z*A common module for NVIDIA Riva Runnables.N)Enum) TYPE_CHECKINGAnyAsyncGenerator AsyncIteratorDict GeneratorIteratorListOptionalTupleUnioncast) AnyMessage BaseMessage) PromptValue)RunnableConfigRunnableSerializable) AnyHttpUrl BaseModelField parse_obj_asroot_validator validator?i) .!?¡¿returnc^ ddl}|jS#t$r}td|d}~wwxYw)z5Import the riva client and raise an error on failure.rNziCould not import the NVIDIA Riva client library. Please install it with `pip install nvidia-riva-client`.) riva.client ImportErrorclient)rivaerrs h/var/www/html/test/engine/venv/lib/python3.12/site-packages/langchain_community/utilities/nvidia_riva.py_import_riva_clientr*1s@ ;;  G  s , ',ceZdZdZy) SentinelTzAn empty Sentinel type.N)__name__ __module__ __qualname____doc__r)r,r,>s!r2r,cTeZdZdZdZdZdZdZdZdZ e de d dfd Z e d d Zy )RivaAudioEncodingaAn enum of the possible choices for Riva audio encoding. The list of types exposed by the Riva GRPC Protobuf files can be found with the following commands: ```python import riva.client print(riva.client.AudioEncoding.keys()) # noqa: T201 ``` ALAWENCODING_UNSPECIFIEDFLAC LINEAR_PCMMULAWOGGOPUS format_coder"c |j|j|jd|S#t$r}t d||d}~wwxYw)zReturn the audio encoding specified by the format code in the wave file. ref: https://mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/WAVE.html )z>The following wave file format code is not supported by Riva: N)r8r5r9KeyErrorNotImplementedError)clsr;r(s r)from_wave_format_codez'RivaAudioEncoding.from_wave_format_codeXsV  ~~#((syyA+N N %**58  s&) AAAcBt}t|j|S)z-Returns the Riva API object for the encoding.)r*getattr AudioEncodingself riva_clients r)riva_pb2zRivaAudioEncoding.riva_pb2fs*+ {00$77r2N)r"zriva.client.AudioEncoding)r-r.r/r0r5r6r7r8r9r: classmethodintrCpropertyrJr1r2r)r4r4Fs] D1 DJ EG  8K  88r2r4ceZdZUdZeeddddgZeeefe d<edd Z e ee d <e dd Z edd d eded efdZy) RivaAuthMixinzBConfiguration for the authentication to a Riva service connection.zhttp://localhost:50051z1The full URL where the Riva service can be found.z"https://user@pass:riva.example.com) descriptionexamplesurlNz@A full path to the file where Riva's public ssl key can be read.rPssl_certr"ct}tt|j}|jdk(}t |jj dd}|j|j||S)z!Return a riva client auth object.https/)rTuse_ssluri) r*rrrRschemestrsplitAuthrT)rHrIrRrY url_no_schemes r)authzRivaAuthMixin.authzsj*+ :txx(**'DHH ++C03 ]]G   r2T)pre allow_reusevalc~t|trttt t|Stt|S)z:Do some initial conversations for the URL before checking.) isinstancer\rrr)rBrcs r) _validate_urlzRivaAuthMixin._validate_urls/ c3  LS$AB BJ$$r2)r"zriva.client.Auth)r-r.r/r0rrrRrr\__annotations__rTr rMr`rrKrrfr1r2r)rOrOmsL"'+,G*,PQ#Cz3  $ VHhsm   u$D1%% %2%r2rOceZdZUdZeej dZeed<eddZ e ed<edd Z e ed <y ) RivaCommonConfigMixinz%A collection of common Riva settings.z!The encoding on the audio stream.)defaultrPencodingi@z*The sample rate frequency of audio stream.sample_rate_hertzzen-USzaThe [BCP-47 language code](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) for the target language. language_codeN) r-r.r/r0rr4r8rkrgrlrLrmr\r1r2r)ririsX/"'!,,7#H#"Ns #M3r2riceZdZUdZej ed<ej ed<d dZd dZ d dZ de fd Z d d Z d d Zy) _Eventz3A combined event that is threadsafe and async safe._event_aeventr"Nchtj|_tj|_y)zInitialize the event.N) threadingEventrpasynciorqrHs r)__init__z_Event.__init__soo' }} r2cl|jj|jjyzSet the event.N)rpsetrqrvs r)rzz _Event.sets   r2cl|jj|jjyry)rpclearrqrvs r)r|z _Event.clears"  r2c6|jjS)zIndicate if the event is set.)rpis_setrvs r)r~z _Event.is_sets{{!!##r2c8|jjy)zWait for the event to be set.N)rpwaitrvs r)rz _Event.waits r2cTK|jjd{y7w)z#Async wait for the event to be set.N)rqrrvs r) async_waitz_Event.async_waitsll!!!s (&(r"N)r-r.r/r0rsrtrgrurwrzr|boolr~rrr1r2r)roros>= OO ]]'   $$"r2rooutput_directory sample_ratec|r{tjddd|5}|j}dddtjd}|j d|j d|j|||fSy #1swYWxYw) zECreate a new wave file and return the wave write object and filename.bxz.wavF)modesuffixdeletedirNwbr=rX)NN)tempfileNamedTemporaryFilenamewaveopen setnchannels setsampwidth setframerate)rrf wav_file_namewav_files r) _mk_wave_filers  ( (fU8H  # FFM #99]D1a a k*x((  # #s A??Brc TTSInputTypect|tr|jSt|trt |j St |S)zAttempt to coerce the input value to a string. This is particularly useful for converting LangChain message to strings. )rer to_stringrr\content)rcs r)_coerce_stringrs< #{#}}#{#3;; s8Or2inputsc#4Kd}|D]}t|}tD]-}||vs|j|d\}}||z|zd}||vr&/||z }t|tkDs\t dt|tD] }|||dzd}|r|yyw)z9Filter the input chunks are return strings ready for TTS.r=rN)r_SENTENCE_TERMINATORSr]len_MAX_TEXT_LENGTHrange)rbufferchunk terminator last_sentenceidxs r)_process_chunksrs Fu%0 J%',{{:q'A$ u},z99%  % v;) )QF -=> ,S37++ ,F!& s B%B B#5BceZdZUdZej ed<ejed<ejed<e ed<e ed<e ed<e ejed<dd e d d fd Z d eed d ffd Zd eefdZed efdZed efdZed efdZed efdZddede e d d fdZddede e d d fdZdde e d d fdZdde e d d fdZdedd d fdZy ) AudioStreamz%A message containing streaming audio. _put_lock_queueoutputhangup user_talking user_quiet_workermaxsizer"Nctj|_tj||_tj|_t|_t|_ t|_ d|_ y)zInitialize the queue.)rN) rsLockrqueueQueuerrrorrrr)rHrs r)rwzAudioStream.__init__ sP")kk'2 kkm h  ("H r2c#K |jjdt}|t k(ry||jj J#tj$rY`wxYww)zReturn an error.TN)rget_QUEUE_GET_TIMEOUTrEmptyHANGUP task_donerHnext_vals r)__iter__zAudioStream.__iter__sc ;;??41CD 6!N KK ! ! #;;  s'A' A)A'A$!A'#A$$A'c$K tjjd|jjdt d{}|tk(ry||jjq7.#t j$rYwxYww)z4Iterate through all items in the queue until HANGUP.TN) ruget_event_looprun_in_executorrrrrrrrrs r) __aiter__zAudioStream.__aiter__&s !(!7!7!9!I!I$++//41C"6!N KK ! ! #;;  s;BAA7A5A7 *B5A77B  B B  Bc6|jjS)z(Indicate if the audio stream has hungup.)rr~rvs r)hungupzAudioStream.hungup9s{{!!##r2c6|jjS)z-Indicate in the input stream buffer is empty.)remptyrvs r)rzAudioStream.empty>s{{  ""r2c|jxr |j}|jduxr7|jj xr|jj}|xr|S)z;Indicate if the audio stream has hungup and been processed.N)rrris_aliver)rH input_done output_dones r)completezAudioStream.completeCsa[[/TZZ LL $ $LL))++ $ !!#  )k)r2cP|jr|jjSy)z&Indicate if the ASR stream is running.F)rrrvs r)runningzAudioStream.runningNs! <<<<((* *r2itemtimeoutc|j5|jr td|tur|jj |j j||dddy#1swYyxYw)zPut a new item into the queue.z?The audio stream has already been hungup. Cannot put more data.rN)rr RuntimeErrorrrrzrput)rHrrs r)rzAudioStream.putUs` ^^ 3{{"Uv~ ! KKOOD'O 2 3 3 3s AA--A6cKtj}tj|jd|j||d{y7w)z$Async put a new item into the queue.N)rurwait_forrr)rHrrloops r)aputzAudioStream.aput`s=%%'t33D$((DI7SSSsA A A Ac0|jt|y)zSend the hangup signal.N)rrrHrs r)closezAudioStream.closees !r2cLK|jt|d{y7w)zAsync send the hangup signal.N)rrrs r)aclosezAudioStream.acloseisii(((s $"$ responseszrasr.StreamingRecognizeResponsec"jr tdtjddd fd }tj|_dj _j jjy) zIDrain the responses from the provided iterator and put them into a queue.z,An ASR instance has already been registered.rXrrNc2jD]}|js|jD]}|js|jrwjj j jtt|jdj}jj|jjrjjj j y)zConsume the ASR Generator.rN)rresults alternativesis_finalrr|rrzrr\ transcriptrrr~)responseresultr has_startedrrHs r)workerz$AudioStream.register..workerts    % 0''&.. 0F!.. ))//1++-%)#v/B/B1/E/P/P%Q   3!..557))--/--/ 0  0r2)targetTr) rrrsBarrierThreadrdaemonstartr)rHrrrs`` @r)registerzAudioStream.registermsi <<MN N''15  0*!''v6 "  r2)rN) r-r.r/r0rsrrgrrror rrLrwr bytesrrStreamInputTyperrMrrrrrrrrrr rr1r2r)rrsv/~~ KK KK N i&& ''D$)E4$56$"$!?$&$$$#t##*$** 3 3(3- 34 3TT#TRVT "Xc]"d")HSM)T)(+L"MRVr2rc eZdZUdZdZeed<dZeed<eddZ e ed <ed d Z e ed <ed d Z e ed<ed edeeefdeeeffdZeddZddZ ddedeededefdZy)RivaASRzNA runnable that performs Automatic Speech Recognition (ASR) using NVIDIA Riva.nvidia_riva_asrrzA Runnable for converting audio bytes to a string.This is useful for feeding an audio stream into a chain andpreprocessing that audio to create an LLM prompt.rPr=z7The number of audio channels in the input audio stream.rSaudio_channel_countTz\Controls whether or not Riva should attempt to filter profanity out of the transcribed text.profanity_filterz]Controls whether Riva should attempt to correct senetence puncuation in the transcribed text.enable_automatic_punctuationravaluesr"ct}|Sz4Validate the Python environment and input arguments.r*rBr_s r)_validate_environmentzRivaASR._validate_environment ! r2c t}|jd|j|j|j|j d|j |j|jS)z)Create and return the riva config object.Tr=)rkrlrmax_alternativesrrrm)interim_resultsconfig) r*StreamingRecognitionConfigRecognitionConfigrkrlrrrrmrGs r)rzRivaASR.configsq*+ 55 00"&"8"8$($<$|j}|j||j}|j |g}|j s|j j5|j jjd}dddr|j jsT ||j jgz }|j j|j jsTtjdt!|dj#|j%S|j sy#1swYxYw#tj$rYwxYw)z3Transcribe the audio bytes into a string with Riva.) audio_chunksstreaming_configg?NzRiva ASR returning: %s r)rrstreaming_response_generatorrrrr not_emptyrr get_nowaitrrr_LOGGERdebugreprjoinstrip)rHrrrservicer full_responsereadys r)invokezRivaASR.invokes;}}'')G<<"!%=I NN9 %$& ..'' 9 ..33C8 9,,,,.!%%,,*A*A*C)DD LL**,  ,,,,.  6]8KLxx .4466.. 9 9!;;! !s/&E:E%E"%E;:E;)r"z&riva.client.StreamingRecognitionConfig)r"zriva.client.ASRServicer)r-r.r/r0rr\rgrPrrrLrrrrrKrrrrMrr ASRInputTyper r ASROutputTyper$r1r2r)rrs Y!D#! < % P #  5d*/  <* $4S>d38n   ,0  (    r2rc |eZdZUdZdZeed<dZeed<eddZ eed <ed d Z e eed <e d e deeefdeeeffdZed e dedefdZddZ ddede ededefdZ ddeede ede edeefdZ ddeede ede edeed ffdZy )RivaTTSz?A runnable that performs Text-to-Speech (TTS) with NVIDIA Riva.nvidia_riva_ttsrz_A tool for converting text to speech.This is useful for converting LLM output into audio bytes.rPzEnglish-US.Female-1zThe voice model in Riva to use for speech. Pre-trained models are documented in [the Riva documentation](https://docs.nvidia.com/deeplearning/riva/user-guide/docs/tts/tts-overview.html).rS voice_nameNzThe directory where all audio files should be saved. A null value indicates that wave files should not be saved. This is useful for debugging purposes.rTrrr"ct}|Srrrs r)rzRivaTTS._validate_environment rr2vc|rAtj|}|jddt|j S|S)NT)parentsexist_ok)pathlibPathmkdirr\absolute)rBr,dirpaths r)_output_directory_validatorz#RivaTTS._output_directory_validator's= ll1oG MM$M 6w'')* *r2ct} |j|jS#t$r}t d|d}~wwxYw)r z5Error raised while connecting to the Riva TTS server.N)r*SpeechSynthesisServicer`rrrs r)rzRivaTTS._get_service0sG)+  55dii@ @ G  rrrrc Vdj|jt|gS)zDPerform TTS by taking a string and outputting the entire audio file.r2)r transformiter)rHrrrs r)r$zRivaTTS.invoke:s"xxtUG}566r2c+K|j}t|j|j\}}t |D]}t j d||j||j|j|jj|j}|D]3} tt| j} |r|j| | 5|r'|j!t j d|yyw)zHPerform TTS by taking a stream of characters and streaming output bytes.zRiva TTS chunk: %s)textr*rmrksample_rate_hzzRiva TTS wrote file: %sN)rrrrlrrrsynthesize_onliner*rmrkrJrraudiowriteframesrawr) rHrrrr!rrrrrespr?s r)r9zRivaTTS.transformCs##%#0  ! !4#9#9#  x %U+ E MM. 6 11??"00//#55 2I" UDJJ/++E2   (  NN  MM3] C sDDc P Ktj tj tj d fd }dtt f fd d fd d fd } j |} j |} tj jdd{} j|turn|L|d{|d{y78#tjj$rYwxYw747,w) zGIntercept async transforms and route them to the synchronous transform.r"NcxK23d{}j|76jtyw)z#Produce input into the input queue.N) put_nowait_TRANSFORM_END)rcr input_queues r) _producerz%RivaTTS.atransform.._producerus:" , ,c&&s+ ,U  " "> 2s:#!#:#:c3K jd}|tk(ry|"#tj$rY8wxYww)zIterate over the input_queue.rrN)rrrrE)rcrFs r)_input_iteratorz+RivaTTS.atransform.._input_iterator{sM%//#/6C.( {{s A'A=A=AcjD]}j|jty)z!Consume the input with transform.N)r9rDrE)rcrI out_queuerHs r) _consumerz%RivaTTS.atransform.._consumers:~~o&78 *$$S) *   0r2cFKjdd{y7w)z"Coroutine that wraps the consumer.N)r)rLrsr)_consumer_coroz*RivaTTS.atransform.._consumer_coros&&tY7 7 7s !!rr) ruget_running_looprrr r create_taskrr exceptions TimeoutErrorrrE)rHrrrrGrNproducerconsumerrcrLrIrFrrKs`` @@@@@r) atransformzRivaTTS.atransformjs'')#(;;= #*==?  3 ,!7  1  8##IK0##N$45 #,,Y]]_cBB    !n$IC%%22   sZBD&'C?C=C? $D&-D". D&7D$8D&=C??DD&DD&$D&)r"z"riva.client.SpeechSynthesisServicer)r-r.r/r0rr\rgrPrr*rr rrKrrrrr5rrr TTSOutputTyper$r r9rrrUr1r2r)r(r(s J!D#! E  aJ',  5'hsm4S>d38n !"CC#,077(7 7  7,0%D %%D(%D3- %D - %DT,05\*5(53- 5  t+ , 5r2r()r"z riva.client)Nr0ruloggingr0rrrsrenumrtypingrrrrrr r r r r rrlangchain_core.messagesrrlangchain_core.prompt_valuesrlangchain_core.runnablesrrpydanticrrrrrrr$r'riva.client.proto.riva_asr_pb2r&proto riva_asr_pb2rasr getLoggerr-rrrrr*r,rrEr\r4rOrirofloat Wave_writerrrrrStreamOutputTyperr%r&rrrVr( NVIDIARivaASR NVIDIARivaTTSNVIDIARivaStreamr1r2r)ris0      <4I11 '  H %9 "" $8T$8N%I%BI(""Dsm27 8C=(4??3 34"  3 H^43d?9S6y()LL`  c}45cPS*k12  _}45_F  r2