7|h3ddlmZddlZddlmZmZmZmZmZm Z m Z m Z m Z m Z ddlZddlZddlmZmZddlmZddlmZerddlmZmZddlmZGd d eZy) ) annotationsN) TYPE_CHECKINGAny AsyncIteratorDict GeneratorIteratorListMappingOptionalUnion)AsyncCallbackManagerForLLMRunCallbackManagerForLLMRun)LLM)GenerationChunk)RESTfulChatModelHandleRESTfulGenerateModelHandle)LlamaCppGenerateConfigc`eZdZUdZdZded<ded< ded< ded < d dfd Zedd Zedd Z dd Z d ddZ d ddZ d ddZ d ddZe d dZ d d!dZ d d"dZxZS)# Xinferencea `Xinference` large-scale model inference service. To use, you should have the xinference library installed: .. code-block:: bash pip install "xinference[all]" If you're simply using the services provided by Xinference, you can utilize the xinference_client package: .. code-block:: bash pip install xinference_client Check out: https://github.com/xorbitsai/inference To run, you need to start a Xinference supervisor on one server and Xinference workers on the other servers Example: To start a local instance of Xinference, run .. code-block:: bash $ xinference You can also deploy Xinference in a distributed cluster. Here are the steps: Starting the supervisor: .. code-block:: bash $ xinference-supervisor Starting the worker: .. code-block:: bash $ xinference-worker Then, launch a model using command line interface (CLI). Example: .. code-block:: bash $ xinference launch -n orca -s 3 -q q4_0 It will return a model UID. Then, you can use Xinference with LangChain. Example: .. code-block:: python from langchain_community.llms import Xinference llm = Xinference( server_url="http://0.0.0.0:9997", model_uid = {model_uid} # replace model_uid with the model UID return from launching the model ) llm.invoke( prompt="Q: where can we visit in the capital of France? A:", generate_config={"max_tokens": 1024, "stream": True}, ) Example: .. code-block:: python from langchain_community.llms import Xinference from langchain.prompts import PromptTemplate llm = Xinference( server_url="http://0.0.0.0:9997", model_uid={model_uid}, # replace model_uid with the model UID return from launching the model stream=True ) prompt = PromptTemplate( input=['country'], template="Q: where can we visit in the capital of {country}? A:" ) chain = prompt | llm chain.stream(input={'country': 'France'}) To view all the supported builtin models, run: .. code-block:: bash $ xinference list --all Nz Optional[Any]client Optional[str] server_url model_uidzDict[str, Any] model_kwargsc  ddlm}|xsi}t |d i|||d|j td|j tdi|_ d|_ |j||jrd||jd <||||_ y#t$r( ddlm}n#t$r}td|d}~wwxYwYwxYw) Nr) RESTfulClientzCould not import RESTfulClient from xinference. Please install it with `pip install xinference` or `pip install xinference_client`.rrrzPlease provide server URLzPlease provide the model UIDFzBearer Authorization) xinference.clientr ImportErrorxinference_clientsuper__init__r ValueErrorr_headers_cluster_authed_check_cluster_authenticatedr)selfrrapi_keyrre __class__s b/var/www/html/test/engine/venv/lib/python3.12/site-packages/langchain_community/llms/xinference.pyr%zXinference.__init__s  7$)r   (& ,  ?? "89 9 >> !;< <(* $ ))+  4#7#7/6wi-@DMM/ *#J8 ?  ; !Y < s5B C'B.-C. C7 CCC Ccy)zReturn type of llm. xinferencer r*s r. _llm_typezXinference._llm_typescZid|jid|jid|jiS)zGet the identifying parameters.rrrrr1s r._identifying_paramszXinference._identifying_paramssC T__- DNN+ t001  r3c|jd}tj|}|jdk(rd|_y|jdk7rt d|j d|j }t|d|_y)Nz/v1/cluster/authFz+Failed to get cluster information, detail: detailauth)rrequestsget status_coder( RuntimeErrorjsonbool)r*urlresponse response_datas r.r)z'Xinference._check_cluster_authenticateds!!12<<$   3 &#(D ##s*"'}}x89;%MMOM#' f(=#>D r3c t|j td|jj|j}|j di}i|j |}|r||d<|r4|j dr#d}|j ||||D]}||z } |S|j||} | dd d S) aqCall the xinference model and return the output. Args: prompt: The prompt to use for generation. stop: Optional list of stop words to use when generating. generate_config: Optional dictionary for the configuration used for generation. Returns: The generated string by the model. Client is not initialized!generate_configstopstream)modelprompt run_managerrFrKrFchoicesrtext)rr& get_modelrr<r_stream_generategenerate) r*rKrGrLkwargsrJrFcombined_text_outputtoken completions r._callzXinference._calls$ ;; 9: : %%dnn54:JJ?PRT4UBT..B/B &*OF # 228<#% ..' / / . %-$  .( 'vWJi(+F3 3r3c#FK|j||}|D]}t|ts|jdg}|s)|d}t|ts?|jdd} |jd} |r|j | |j | | yw) a^ Args: prompt: The prompt to use for generation. model: The model used for generation. stop: Optional list of stop words to use when generating. generate_config: Optional dictionary for the configuration used for generation. Yields: A string token. rMrNrrOrIlogprobs)rUverbose log_probsN)rR isinstancedictr<on_llm_new_tokenrZ) r*rJrKrLrFstreaming_responsechunkrNchoicerUr[s r.rQzXinference._stream_generates$#^^?, ( $E%&))Ir2$QZF!&$/ & 62 6$*JJz$: &'88&+T\\Y9$  $s)B!B!B!A B!c+K|jdi}i|j|}|r||d<|j||D]C}|s|j|}|r'|j |j |j |EywNrFrG)rZ)r<r_create_generate_stream$_stream_response_to_generation_chunkr^rOrZr*rKrGrLrSrF stream_respr`s r._streamzXinference._streams!**%6;BT..B/B &*OF #77P KAA+N00 $ 1  s ABABc#K|j td|jj|j}|j ||Ed{y7w)NrErM)rr&rPrrR)r*rKrFrJs r.rdz"Xinference._create_generate_stream.sL ;; 9: : %%dnn5>>>QQQsAAAAc Pd}t|tr|jdg}|rj|d}t|trJ|jdd}t|t|jdd|jdd St d t| St d ) z0Convert a stream response to a generation chunk.rIrNrrO finish_reasonNrY)rkrY)rOgeneration_infozchoice type error!)rOzstream_response type error!)r\r]r<r TypeError)stream_responserUrNras r.rez/Xinference._stream_response_to_generation_chunk6s  ot ,%)))R8G fd+"JJvr2E*"(,*0**_d*K%+ZZ D%A)$$899&E229: :r3c 4K|jdi}i|j|}|r||d<|j||23d{}|s |j|}|r/|j |j |j d{|T7O7 6ywrc)r<r_acreate_generate_streamrer^rOrZrfs r._astreamzXinference._astreamQs!**%6;BT..B/B &*OF #!%!>!>v!W  +AA+N%66 $ 7   "Xs@F 88F 0G3F15E77F 8 F1FF1 F FF F1 G*F-+G1G7F: 8G?G)NNN)rrrrr+rrr)returnstr)rzMapping[str, Any])rNone)NN) rKrrGOptional[List[str]]rL"Optional[CallbackManagerForLLMRun]rSrrr) rJz=Union['RESTfulGenerateModelHandle', 'RESTfulChatModelHandle']rKrrLrrFz"Optional['LlamaCppGenerateConfig']rzGenerator[str, None, None]) rKrrGrrLrrSrrzIterator[GenerationChunk])N)rKrrFOptional[Dict[str, List[str]]]rz Iterator[str])rnrrr) rKrrGrrLz'Optional[AsyncCallbackManagerForLLMRun]rSrrzAsyncIterator[GenerationChunk])rKrrFrrzAsyncIterator[str])__name__ __module__ __qualname____doc__r__annotations__r%propertyr2r5r)rWrQrhrd staticmethodrerqrp __classcell__)r-s@r.rrsZx!FM &#  :%)#'!% (9!(9!(9 (9  (9T   ?"%):> *4*4"*48 *4  *4 *4`;?>B !$L!$!$8 !$ < !$ $ !$L%):> "8    # ,NRRR,JR R;; ;;:%)?C "=    ( ,NR#7#7,J#7 #7r3r) __future__rr?typingrrrrrr r r r r rtr;langchain_core.callbacksrr#langchain_core.language_models.llmsrlangchain_core.outputsrr!rrxinference.model.llm.corerrr r3r.rsF"    42T@j7j7r3