9|hFdZddlmZddlZddlZddlZddlZddlZddl Z ddl m Z m Z ddl m Z mZmZmZmZmZmZmZmZddlmZddlmZddlmZdd lmZmZdd lm Z m!Z!dd l"m#Z#m$Z$m%Z%dd l"m&Z'dd l"m(Z)ddl*m+Z+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2m3Z3ddl4m5Z5m6Z6ddl4m7Z8ddl9m:Z:m;Z;ddlZ>m?Z?m@Z@mAZAddlBmCZCddlDmEZEddlFmGZGddlHmIZIddlJmKZKddlLmMZMmNZNmOZOddlPmQZRdd lSm&ZTddlSmUZUmVZVe rddlWZXejeZZ[eegeeIe#ffeee\gefe#eIfZ]eegeeIe#ffefZ^Gdde_Z`Gd d!e\ZaGd"d#e\Zb dM dNd$ZcdOd%ZdGd&d'eGZedPd(Zf dQd)Zg dRd*Zh dSd+Zi dTd,Zj dUd-Zk dVd.Zl dWd/Zm dXd0Zn dYd1Zo dZd2Zpddddd3 d[d4Zqdddd5 d\d6Zrdd7 d]d8Zsdddd5 d^d9Ztdddd5 d_d:Zudd7 d]d;Zv d` dad<ZwGd=d>eGd?@ZxejGdAdBZzdbdCZ{dcdDZ|dEZ}dddFddd?ddG dddHZ~dddFddd?ddG dddIZdJZee_ejdKdLe~_y)ez>Utilities for running language models or Chains over datasets.) annotationsN)datetimetimezone) TYPE_CHECKINGAnyCallableDictListOptionalTupleUnioncast)warn_deprecated) Callbacks)BaseLanguageModel) BaseMessagemessages_from_dict) ChatResult LLMResult)RunnableRunnableConfigRunnableLambdaconfig)utils)EvaluatorCallbackHandlerwait_for_all_evaluators)LangChainTracer)Client) get_git_infoget_langchain_env_var_metadata)EvaluationResult RunEvaluator) run_evaluator) as_runnableis_traceable_function)DatasetDataTypeExampleRun TracerSession)LangSmithError) HTTPError) TypedDict)Chain)load_evaluator) EvaluatorTypePairwiseStringEvaluatorStringEvaluator) evaluation)name_generationprogressceZdZdZy)InputFormatErrorz(Raised when the input format is invalid.N)__name__ __module__ __qualname____doc__f/var/www/html/test/engine/venv/lib/python3.12/site-packages/langchain/smith/evaluation/runner_utils.pyr8r8Ns2r>r8c$eZdZdZ ddZddZy) TestResultz1A dictionary of the results of a single test run.c|j}|jDcgc];}|jds&|jds|dvs|jdr|=}}|jdj |dScc}w) zReturn quantiles for the feedback scores. This method calculates and prints the quantiles for the feedback scores across all feedback keys. Returns: A DataFrame containing the quantiles for each feedback key. inputs.outputs.>inputoutput referenceall)include)axis) to_dataframecolumns startswithdescribedrop)selfdfcolto_drops r?get_aggregate_feedbackz!TestResult.get_aggregate_feedbackXs   zz ~~i(~~j)))~~k*   {{5{)..wQ.?? sABc  ddl}g}g}|djD]e\}}|d}|jd}t |t r'|jD cic] \} } d| |  } } } n |i} nd|i} i|djD cic] \} } d | |  c} } | } d |vrUt |d t r:| j |d jD cic] \} } d | |  c} } n|d | d <| j i|D cic]} d | j| jc} |jd |d|jdd|j| |j|h|j||S#t$r}td|d}~wwxYwcc} } wcc} } wcc} } wcc} w)z#Convert the results to a dataframe.rNzfPandas is required to convert the results to a dataframe. to install pandas, run `pip install pandas`.resultsfeedbackrFrDrErCrGz reference.z feedback.Errorexecution_timerun_id)errorrZr[)index) pandas ImportErroritemsget isinstancedictupdatekeyscoreappend DataFrame)rQpdeindicesrecords example_idresultrXoutput_kvrFrfs r?rLzTestResult.to_dataframeos  "&y/"7"7"9 ' Jj)Hjj*G'4(8? H1HQC.!+HH"G,06w0E0E0GH1WQC=!#HAf$f[148HH9? 9L9R9R9TUA:aS)1,U&,K%8AkN HH=EF155'*AGG3F#ZZ0&,-=&>$jj2   NN1  NN: &= '@r||G733Q @  II V Gs/FF7F= G !G  F4# F//F4N)return pd.DataFrame)r9r:r;r<rUrLr=r>r?rArAUs;@ @.,4r>rAc,eZdZdZdfd ZddZxZS) EvalErrorz"Your architecture raised an error.c (t|dd|i|y)NrYr=)super__init__)rQrYkwargs __class__s r?rzzEvalError.__init__s /u//r>cF ||S#t$rtd|dwxYw)Nz%'EvalError' object has no attribute '')KeyErrorAttributeError)rQnames r? __getattr__zEvalError.__getattr__s8 R:  R #Ha!PQ Q Rs )rY BaseExceptionr{rrtNone)rstrrtr)r9r:r;r<rzr __classcell__)r|s@r?rwrws,0Rr>rwc P t|tr^|jj}|j5jjj}t d|d|d|dfdSt|t r|St|tr| fdSt|rt|rttt| fdS |}tt|t|t r|Sttt|rttt| fd St|tsfd SS|S#t$rQtt|}tj|}t j#d|d t%| fd cYSwxYw) zForgive the user if they pass in a chain without memory instead of a chain factory. It's a common mistake. Raise a more helpful error message as well.a$Cannot directly evaluate a chain with stateful memory. To evaluate this chain, pass in a chain constructor that initializes fresh memory each time it is called. This will safegaurd against information leakage between dataset examples. For example: def chain_constructor(): new_memory = z(...) return z*(memory=new_memory, ...) run_on_dataset("z", chain_constructor, ...)cSNr=)chainsr?z(_wrap_in_chain_factory..sur>cSrr=)lcfsr?rz(_wrap_in_chain_factory..ssr>cSrr= runnable_sr?rz(_wrap_in_chain_factory..9r>zWrapping function z as RunnableLambda.cSrr=)wrappedsr?rz(_wrap_in_chain_factory..s7r>cSrr=rsr?rz(_wrap_in_chain_factory..rr>ctSr)r) constructorsr?rz(_wrap_in_chain_factory..s >+6r>)rbr/r|r9memory ValueErrorrrcallabler&r%rr TypeErrorinspect signatureloggerinfor) llm_or_chain_factory dataset_name chain_class memory_class_model user_funcsigrrrrrs @@@@@r?_wrap_in_chain_factoryrs &.$oo..  & & 2 <<11::L$%1>2)]##/.0J L   (*; <## (( 3" & ' !5 6#D3G$HII$ $ #)+F8%9: f/ 0M "4 :#D6$:;I$ $FH-6 6  + #X';t|dts$tdt|dj|dg}nd|vrRt|dt rt d|dDs$tdt|dj|d}nt|dk(ritt|j}t|tr|g}nAt|t rt d|Dr|}ntd |td |t|dk(r|d Std t|d )zGet prompt from inputs. Args: inputs: The input dictionary. Returns: A string prompt. Raises: InputFormatError: If the input format is invalid. Inputs should not be empty.promptz"Expected string for 'prompt', got promptsc3<K|]}t|tywrrbr.0is r? z_get_prompt..s> #$Jq# > z,Expected list of strings for 'prompts', got rJc3<K|]}t|tywrrrs r?rz_get_prompt.. s.Saz!S/A.Srz)LLM Run expects string prompt input. Got z5LLM Run expects 'prompt' or 'prompts' in inputs. Got rz)LLM Run expects single prompt input. Got z prompts.) r8rbrtyper9listrHlennextitervalues)inputsrprompt_s r? _get_promptrs~ <==G6&*C0"4T&:J5K5T5T4UV (#$ f &+T2#> (.y(9> ; #VI./889; # V tFMMO,- gs #iG  &3.S7.S+SG"%Nvh#WX XCF8 L   7|qqz7G ~Y O  r>ceZdZUdZded<y)ChatModelInputzVInput for a chat model. Parameters: messages: List of chat messages. zList[BaseMessage]messagesNr9r:r;r<__annotations__r=r>r?rrs  r>rc|s td|j}d|vr|jd|d<n3t|dk(r%t t |j |d<d|vrV|d}t|trtd|Dr|g}t|dk(rt|d|d<|Stdtd|) zGet Chat Messages from inputs. Args: inputs: The input dictionary. Returns: A list of chat messages. Raises: InputFormatError: If the input format is invalid. rrrErJc3<K|]}t|tywr)rbrcrs r?rz _get_messages..8s2 $%Jq$ 2 rrzGBatch messages not supported. Please provide a single list of messages.zMChat Run expects single List[dict] or List[List[dict]] 'messages' input. Got ) r8copypoprrrrrbrrHr)r input_copy raw_messagess r? _get_messagesr$s <==JV(nnZ8 7 V "4 #89 7*!'* lD )c2 )52 / )>L |  !"4\!_"EJw   #,  !( $  r>c|ra||j}t|ts>t|trt d|Dst d|dt |dyy t|jy#t $r? t|jYy#t $rt d|jdwxYwwxYw)Nc3<K|]}t|tywrrbrrmsgs r?rz>_validate_example_inputs_for_language_model..TsISJsK0IrzWhen using an input_mapper to prepare dataset example inputs for an LLM or chat model, the output must a single string or a list of chat messages. Got: of type .zvExample inputs do not match language model input format. Expected a dictionary with messages or a single prompt. Got: z Please update your dataset OR provide an input_mapper to convert the example.inputs to a compatible format for the llm or chat model you wish to evaluate.) rrbrrrHr8rrr) first_example input_mapper prompt_inputs r?+_validate_example_inputs_for_language_modelrLs#M$8$89 ,, |T *ILII"'yl1C0DAG J-   ,, -  m223# &*112GG  s%A;; CB#B??Cc&|r||j}t|jj|}t |t st d|dt|d|r)t d|jd|jy|j}t|jj|}t|dk(rt|jdk(ry|r)t d|jd|jy) zct|tr t||y|}t|trt |||yt|t rt jd|yy)z9Validate that the example inputs are valid for the model.zSkipping input validation for N)rbrrr/rrrdebug)examplerrrs r?_validate_example_inputsrs[ &(9:3G\J$& eU # .w| L x ( LL9%A B)r>c :|rt|trd\}}d}nEd}|}t|tr |jnd}t|tr |jnd}t ||||dj rt|dj nd||}|Sd}|S)z&0 #J HH(*E-7u-E))4J/9%/G%++TK-   )1!)<)cd}|jr0|j}|r ||vrtjd|d|d|S|rt|dk(r|d}|S|'t|dkDrtjd|d|S)Nz Input key z% not in chain's specified input keys '. Evaluation behavior may be undefined.rJrz#Chain expects multiple input keys: z, Evaluator is likely to fail. Evaluation behavior may be undefined. Specify an input_key in the RunEvalConfig to avoid this warning.) input_keyrwarningr)rrrs r?_determine_input_keyrsI $$ ):5 NNYK()l*QS   J1,qM    C Oa$71*>P P r>cd}|jr0|j}|r ||vrtjd|d|d|S|rt|dk(r|d}|S|'t|dkDrtjd|d|S)NzPrediction key z& not in chain's specified output keys rrJrz$Chain expects multiple output keys: zl, Evaluation behavior may be undefined. Specify a prediction_key in the RunEvalConfig to avoid this warning.)prediction_keyrrr)rrrs r?_determine_prediction_keyrsN .. >< NN!.!12 +},SU   [)Q.$Q   S%5%92;-@; ; r>c|jr%|j}|r||vrtd|d||S|rt|dk(rt|d}|Sd}|S)NzReference key z! not in Dataset example outputs: rJr) reference_keyrrr)rexample_outputsrs r?_determine_reference_keyrs,, }OC 0%%4$57   S1Q6_-a0   r>c nt|tr|St|ttfr5t|ts t|}t ||}|j } nt|t jrd|i|j} t |jfi| }|jj } t|t jr^|jxs|}|jxs|}|jxs|}n-t|r t|St!dt#|t|t$rI|j&r|t!d| d|dt(j*j-||||||| g} | St|t.rt1d| d t1d| d ) N)rrzUnknown evaluator type: zPMust specify reference_key in smith_eval.RunEvalConfig to use evaluator of type z) with dataset with multiple output keys: r)rrrtagszRun evaluator for z is not implemented. PairwiseStringEvaluators compare the outputs of two different models rather than the output of a single model. Did you mean to use a StringEvaluator instead? See: https://python.langchain.com/docs/guides/evaluation/string/z is not implemented)rbr#r1rr0valuesmith_eval_config EvalConfig get_kwargsevaluator_typeSingleKeyEvalConfigrrrrrun_evaluator_decrrr3requires_reference smith_evalStringRunEvaluatorChainfrom_run_and_data_typer2NotImplementedError) eval_configeval_llmrrrrrr evaluator_ eval_type_tagr{r$s r?_construct_run_evaluatorr s+|,+ s34+}5' 4K#KX> #)) K!2!=!= >>[%;%;%=>#K$>$>I&I #2288 k#4#H#H I#--:I(77I>N'55FM +  --3D4E3FGHH*o.  ( (]-B&&3_577F6GqJ  #::QQ   )'R ,  J 7 8! 0Q Q  " /B C  r>cTt||}t||}t||}|||fSr)rrr)rrrrrrrs r? _get_keysr Hs5 %VZ8I.v{CN,V_EM nm 33r>c g}d\}}} |js=|jrCt|jD cgc]} t| tc} rt ||||\}}} |jD]/} t | |j|||| ||} |j| 1|jxsg} | D]}t|tr|j|%t|tr5|jtjj|||||| jt|r|jt|td|d|Scc} w)z Load run evaluators from a configuration. Args: config: Configuration for the run evaluators. Returns: A list of run evaluators. NNN)rrrzUnsupported custom evaluator: z+. Expected RunEvaluator or StringEvaluator.) evaluatorscustom_evaluatorsanyrbr3r r rrgr#rrrrrr)rrrrrrrrrrrjrr$rcustom_evaluators r?rrTs"N/?,I~}   9Q9QRAA/R S3< J _4 0 >=(( - 0  OO         m, -006B- & 5  ! !"2 3 (/ :  ! !22II$'#1"/ J & '  ! !"34D"E F01A0BC=> #, QSsEr callbacksrmetadatac K|x||}t|ts"t|trAtd|Dr/|j |t ||xsg|xsid{St d|d t|}|j |t ||xsg|xsid{}|S7P7#t $rBt|} |jdi| dt ||xsg|xsiid{7}Y|SwxYww) aAsynchronously run the language model. Args: llm: The language model to run. inputs: The input dictionary. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. input_mapper: Optional function to map inputs to the expected format. Returns: The LLMResult or ChatResult. Raises: ValueError: If the LLM type is unsupported. InputFormatError: If the input format is invalid. Nc3<K|]}t|tywrrrs r?rz_arun_llm..OSJsK0Orrrrrz%Input mapper returned invalid format 3 Expected a single string or list of chat messages.rr=) rbrrrHainvokerr8rr) rrrrrrprompt_or_messagesr llm_output llm_inputss r? _arun_llmrsK0)&1 )3 /,d3Or% #&'FG   (F8; %'djb8>r9D93J = 3   &v.J*s{{  %'djb8>r J  sUA'D )B:*D >5B>3B<4B>8D <B>>AD ?DD D D  D rrrc K||n||}t|tr}t|trmt|dk(r_|jrSt t |j}|j|t||xsg|xsid{}|St|xsg||xsi} |j|| d{}|S797w)z%Run a chain asynchronously on inputs.NrJrrrrr) rbr/rcrrrrrrr rrrrrrinputs_valrFrunnable_configs r? _arun_chainr&s%,f,v2FG5% w % LA    4()*}} !#$*"x~2%   M )y8>r }}W_}EE M Fs$BCC2CC CC)rc Kt|trdnd}d} t|tr9t||j|d|d||j dd{}n?|}t ||j|d|d||j dd{}|}|S7H7 #t $rT}tj|d|jd |jd t|t| }Yd}~|Sd}~wwxYww) aAsynchronously run the Chain or language model. Args: example: The example to run. llm_or_chain_factory: The Chain or language model constructor to run. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. input_mapper: Optional function to map the input to the expected format. Returns: A list of outputs. LLMr/Nrrrr failed for example with inputs  rY) rbrrrrar& Exceptionrridreprrw) rrrr chain_or_llmrnrFrrjs r?_arun_llm_or_chainr1s#(02CD'F$ *,= > )$F^ -)J/ !F)*E&F^ -)J/ F M5 $n0 =">>*a   # M$sTD AB*B&?B*B(B*$D &B*(B** D3A D<D DD c |r||}t|ts"t|tr;td|Dr)|j |t ||xsg|xsi}|St d|d t|}|j |t ||xsg|xsi}|S#t $r4t|} |jdi| dt ||xsii}Y|SwxYw) a Run the language model on the example. Args: llm: The language model to run. inputs: The input dictionary. callbacks: The callbacks to use during the run. tags: Optional tags to add to the run. input_mapper: function to map to the inputs dictionary from an Example Returns: The LLMResult or ChatResult. Raises: ValueError: If the LLM type is unsupported. InputFormatError: If the input format is invalid. c3<K|]}t|tywrrrs r?rz_run_llm..Nrrrrz'Input mapper returned invalid format: rr)rrr=) rbrrrHinvokerr8rr) rrrrrrrr llm_promptsrs r?_run_llmr60s%2)&1 )3 /,d3Or3=3J6 )#&'FG  %f-K%'djb8>r$J    &v.J#% HNPRSJ   s62B**9C'&C'c z||n||}t|trut|tret|dk(rW|jrKt t |j}|j|t||xsg|xsi}|St|xsg||xsi} |j|| }|S)zRun a chain on inputs.rJrrr!) rbr/rcrrrrrr4rr"s r? _run_chainr8ns%,f,v2FG5% w % LA    4()* !#$*"x~2  M )y8>r go> Mr>c t|trdnd}d} t|tr1t||j|d|d||j d}n7|}t ||j|d|d||j d}|}|S#t $rc}t|j} tj|d|jd |jd | d | t| }Yd}~|Sd}~wwxYw) a Run the Chain or language model synchronously. Args: example: The example to run. llm_or_chain_factory: The Chain or language model constructor to run. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. Returns: Union[List[dict], List[str], List[LLMResult], List[ChatResult]]: The outputs of the model or chain. r(r/Nrrrrr)r*z Error Type: z , Message: r,) rbrr6rrar8r-rr9rrr.rw) rrrrr0rnrFrrj error_types r?_run_llm_or_chainr;s*02CD'F$ *,= >"${#F^)J/ F)*E{#F^)J/ F M $!W%% n0 =">>*ZL A3 8 # M$sA:B DAC;;Dc t||}|j|}t|j|j|} | st d|d| D cgc]} | j s| j } } | r t| nd} | r| jnd} |xsi}t}|ri|d|i}| |d<|j||j|rd|ini|}|j d|jz}t#d|d|d|d|j d|||| fScc} w#tt tf$rE}d t|vr|tj}d |d |d |d }t d|d|d}~wwxYw)N)r) dataset_idas_ofzDataset z has no example rows.gitdataset_versionr)reference_dataset_id project_extrarzalready exists z+ run_on_dataset( ... project_name="z - z", # Update since z already exists ) z Test project z/ already exists. Please use a different name: z/compare?selectedSessions=z)View the evaluation results for project 'z' at: z View all tests for Dataset z at: T)flush)r read_datasetr list_examplesr.r modified_atmax isoformatr create_projectr-r,ruuiduuid4urlprint)clientrr project_nameproject_metadatarr@ wrapped_modeldatasetrexrFmax_modified_atinferred_versiongit_infoprojectrjuid example_msgcomparison_urls r?_prepare_eval_runr[s++?NM!!|!1FGHH,4Gb2>>GKG+6c+&4O6E0024 +1r>  " x   /?*+'' !(,064.b% ( ([[%? |#LLN 3L>B !&&2^6'++ H  '7H 44WH, z> 2  CF *Gjjl .C5(:<.I L>*- !    s&D),D) AD..FAFFc0eZdZUdZded<ded<ded<y) _RowResultz5A dictionary of the results for a single example row.z Optional[List[EvaluationResult]]rXzOptional[float]rZ Optional[str]r[Nrr=r>r?r]r]s?..## r>r]F)totalceZdZUdZded<ded<ded<ded <d ed <d Zd ed< ddZddZddZ ddZ dddZ e d ddZ y )_DatasetRunContainerz3A container to help manage the state of a eval run.rrNr+rWMCFrQ List[Example]rzList[RunnableConfig]configsNz6Optional[List[smith_eval_config.BATCH_EVALUATOR_LIKE]]batch_evaluatorsc `i}t|j|D]\}}tt|j t |j i}|j|j dg|j d|j dd|t |j <t|tr&|j|t |j d<n||t |j d<|js|j|t |j d<|S)NrXrZr[)rErXrZr[rYrFrG) ziprrr]rarr.rrbrwrYr)rQ batch_resultsall_eval_resultsrWrrF row_results r?_merge_test_outputsz(_DatasetRunContainer._merge_test_outputss "4==-@ HOGVj*:*>*>s7::PR*STJ &NN:r:",..1A"B$..2 (GC O $ &),4:LLGJJ(15;GJJ(28?GJJ(5 Hr>c |j}|sgS|jDcgc]}|t|j}}g}tj j 5}|D]} |||j}t|tr|j}|jtt||j|jjfi|d|jjd ddd|Scc}w#t $r.} t"j%dt'|d| Yd} ~ d} ~ wwxYw#1swY|SxYw)N)r[ project_idzError running batch evaluator z: )rerrr. concurrentfuturesThreadPoolExecutorrbr"rcrgrsubmitrNcreate_feedbackrWr-rr\r/) rQrunsrr runs_listaggregate_feedbackexecutor evaluatorrnrjs r?_run_batch_evaluatorsz*_DatasetRunContainer._run_batch_evaluators,s8** I:>--HwT#gjj/*H H    2 2 4 '  &y$--@F!&*:;!'&--d4.@A#HOO 33  $#'<}tt|dD]$}t|trT|j }|j D]4\\}}}|jt|ijd|i6ht|tsy|j} | r3| jr'| j| jz jnd} | rt| jnd} |jt|j ij| | | d| |t|j <'Att"tt$f||fS)NrrX)rZr[run)rdrrrbrlogged_eval_resultsr` setdefaultrrdr latest_runend_time start_time total_secondsr.rmr r]) rQriall_runsccallback eval_results_rmrqrzrZr[s r?_collect_metricsz%_DatasetRunContainer._collect_metricsEsX!# =A q~6 =h(@A#+#?#?L.:.@.@.B*J(33C ORHOO'O /:"--C3<<6EEG!# -0S[TF$//H4G4G0H"MTT.<&,#&:=HS!4!456- = =0Dj)+;c6tjdt|j\}}d}|jr&tjd|j |}|j ||}t|jj||S)Nz#Waiting for evaluators to complete.zRunning session evaluators.)rOrWaggregate_metrics) rrrrrerxrkrArWr)rQrhrirrurWs r?_collect_test_resultsz*_DatasetRunContainer._collect_test_resultsbs  9:!%)%:%:%<"(!  KK5 6!%!;!;H!E **=:JK**0  r>c|j|}|r |j}t| |jj|jjtjtj|S#t$r+}tj dt |Yd}~d}~wwxYw#t$r,}tj dt |Yd}~|Sd}~wwxYw)Nz$Failed to print aggregate feedback: )r~zFailed to close project: )rrU_display_aggregate_resultsr-rrr/rNupdate_projectrWr.rnowrutc)rQrhverboserW agg_feedbackrjs r?finishz_DatasetRunContainer.finishts,,];  O&==? *<8 @ KK & & (,,x||*D '   O CDG9MNN O @ LL4T!WI> ? ? @s/BAB< B9!B44B9< C1!C,,C1c ,|xstj}| r| si} | jd| it||||| || \} } }}|xsg}| jj dxsij D]\}}|jd|d|d| jdi}| r| |d<t|} t| |||jxstj}t|d| |tjt!|}|Dcgc]O}t#t%| j&||j(t+|xsg||j(d |g||| Q}}||| | |||r|j, Sd Scc}w) N revision_id)rPrr@r?zgit:=r@r)rOrNrm)rrNrmmax_concurrency)rrrr)rNrWrQrrdre)r5 random_namerdr[rrar`rgrrrr(kvrr6ProgressBarCallbackrrrrr.rre)clsrNrrrOr4rrconcurrency_levelrPrr@rQrWrRrrprq run_metadatar progress_barrrds r?preparez_DatasetRunContainer.prepares$D'B'B'D ##%  # #]K$@ A4E   -+5 1 wzr%%))%06B==? (DAq KK$qc1# ' ()7+;+;r?rara s%= N  !!OSLS0  ,"2G: D   $":>$(8E E E 5 E $ E 7 E "E 6E E 3E #E 8E  E E r>rac~ ddlm}|}|duxrdtt|vS#t$rYywxYw)Nr) get_ipythonzmqshellF)IPythonrrrr_)rress r?_is_jupyter_environmentrsB'm}D(IZ3tCy>-II s -0 <<ctrddlm}m}||d||y|j dd}t dt |y)Nr)HTMLdisplayz

Experiment Results:

c |dS)Nz.2fr=)xs r?rz,_display_aggregate_results..s aWr>right) float_formatjustifyz Experiment Results:)rIPython.displayrr to_stringrM)aggregate_resultsrrformatted_strings r?rrsQ 1345!",66-w7  &' r>aThe input_mapper argument is deprecated and will be removed in a future release. Please add a RunnableLambda to your chain to map inputs to the expected format instead. Example: def construct_chain(): my_chain = ... input_mapper = {'other_key': 'MyOtherInput', 'my_input_key': x} return input_mapper | my_chain run_on_dataset(..., llm_or_chain_factory=construct_chain) (See https://api.python.langchain.com/en/latest/schema/langchain.schema.runnable.base.RunnableLambda.html)r)r4r@rrOrPrrc K| jdd} | rtdtd| tj d} | jdd} | rtddd| r tdd | j d d |xs t }tj|||||| | ||| | } tj| jd j dgttjt| j | | j"| jd{}| j%||S7w)Nr0.0.305Tmessagependingrr0.1.9qThe tags argument is deprecated and will be removed in a future release. Please specify project_metadata instead.PThe following arguments are deprecated and will be removed in a future release: rrremovalrPrr@rrrrr)rr_INPUT_MAPPER_DEP_WARNINGr!rarrrarrunnable_utilsgather_with_concurrencyrdmap functoolspartialr1rQrr)rNrrr4r@rrOrPrrr{rr containerrhs r?arun_on_datasetrsd::nd3L +DdS46::=I ::fd #D  U   4{{}oQ    vxF$,, )'- I)@@!  !23    "%.%<%<)          M   M7  ;; sD,E.E/Ec N| jdd} | rtdtd| jdd} | rtddd| tj d} | r tdd | j d d |xs t }tj|||||| | ||| | } |d k(rJt| j| jDcgc]\}}t||| j|  }}}ntj| jd 5}t!|j#t%j&t| j| | j| j}ddd| j)|Scc}}w#1swY"xYw)NrrTrrrrrrrrrrrr)rrrr!rarrrarrgrrdr;rQr%get_executor_for_configrrrrr)rNrrr4r@rrOrPrrr{rrrrrrhrvs r?run_on_datasetr8s::nd3L +DdS ::fd #D  U  46::=I   4{{}oQ    vxF$,, )'- IA$'y'9'99;L;L#M   %.%<%<)     4 4Y5F5Fq5I J h  %%)-6-D-D%1 &&%% M    M7  ;;/   s#F%AFF$a1 Run the Chain or language model on a dataset and store traces to the specified project name. Args: dataset_name: Name of the dataset to run the chain on. llm_or_chain_factory: Language model or Chain constructor to run over the dataset. The Chain constructor is used to permit independent calls on each example without carrying over state. evaluation: Configuration for evaluators to run on the results of the chain concurrency_level: The number of async tasks to run concurrently. project_name: Name of the project to store the traces in. Defaults to {dataset_name}-{chain class name}-{datetime}. project_metadata: Optional metadata to add to the project. Useful for storing information the test variant. (prompt version, model version, etc.) client: LangSmith client to use to access the dataset and to log feedback and run traces. verbose: Whether to print progress. tags: Tags to add to each run in the project. revision_id: Optional revision identifier to assign this test run to track the performance of different versions of your system. Returns: A dictionary containing the run's project name and the resulting model outputs. For the (usually faster) async version of this function, see :func:`arun_on_dataset`. Examples -------- .. code-block:: python from langsmith import Client from langchain_openai import ChatOpenAI from langchain.chains import LLMChain from langchain.smith import smith_eval.RunEvalConfig, run_on_dataset # Chains may have memory. Passing in a constructor function lets the # evaluation framework avoid cross-contamination between runs. def construct_chain(): llm = ChatOpenAI(temperature=0) chain = LLMChain.from_string( llm, "What's the answer to {your_input_key}" ) return chain # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum) evaluation_config = smith_eval.RunEvalConfig( evaluators=[ "qa", # "Correctness" against a reference answer "embedding_distance", smith_eval.RunEvalConfig.Criteria("helpfulness"), smith_eval.RunEvalConfig.Criteria({ "fifth-grader-score": "Do you have to be smarter than a fifth grader to answer this question?" }), ] ) client = Client() run_on_dataset( client, dataset_name="", llm_or_chain_factory=construct_chain, evaluation=evaluation_config, ) You can also create custom evaluators by subclassing the :class:`StringEvaluator ` or LangSmith's `RunEvaluator` classes. .. code-block:: python from typing import Optional from langchain.evaluation import StringEvaluator class MyStringEvaluator(StringEvaluator): @property def requires_input(self) -> bool: return False @property def requires_reference(self) -> bool: return True @property def evaluation_name(self) -> str: return "exact_match" def _evaluate_strings(self, prediction, reference=None, input=None, **kwargs) -> dict: return {"score": prediction == reference} evaluation_config = smith_eval.RunEvalConfig( custom_evaluators = [MyStringEvaluator()], ) run_on_dataset( client, dataset_name="", llm_or_chain_factory=construct_chain, evaluation=evaluation_config, ) zrun_on_dataset(zawait arun_on_dataset()z )rrrrrtrb)rDict[str, Any]rtr)rrrtrc)rr)rrrtr)rr)rr/rrrtr)rr)rrbrrrtr) rrbrrcr4rrr(rtzOptional[List[RunEvaluator]])rsmith_eval.RunEvalConfigrrrtr^)rrrrrtr^)rrrrrtr^)rzYUnion[smith_eval_config.SINGLE_EVAL_CONFIG_TYPE, smith_eval_config.CUSTOM_EVALUATOR_TYPE]rzOptional[BaseLanguageModel]rrrr(rrrr^rr^rr^rtr#) rrrrrrrrrtz2Tuple[Optional[str], Optional[str], Optional[str]])rrrrrr(rrrrrrrtzList[RunEvaluator])rrrrrrrrrrrrrtUnion[str, BaseMessage])rUnion[Chain, Runnable]rrrrrrrrrrrtzUnion[dict, str]) rr)rrrrbrrrtz'Union[dict, str, LLMResult, ChatResult])rrrrrrrrrrrrrtr)rrrrrrrrrrrrrtzUnion[Dict, str]r )rNrrrrrrOrrPrrrr@zOptional[Union[str, datetime]]rtz1Tuple[MCF, TracerSession, Dataset, List[Example]])rtr)rrurtr)rNzOptional[Client]rrrrr4rr@rrrrOr^rPrrrrr^r{rrtr)r< __future__rconcurrent.futuresrn dataclassesrrloggingrJrrtypingrrrr r r r r rlangchain_core._apir langchain_core.callbacks.managerrlangchain_core.language_modelsrlangchain_core.messagesrrlangchain_core.outputsrrlangchain_core.runnablesrrrrr%rr!langchain_core.tracers.evaluationrr langchain_core.tracers.langchainrlangsmith.clientr langsmith.envr r!langsmith.evaluationr"r#r$rlangsmith.run_helpersr%r&langsmith.schemasr'r(r)r*r+langsmith.utilsr,requestsr-typing_extensionsr.langchain.chains.baser/langchain.evaluation.loadingr0langchain.evaluation.schemar1r2r3langchain.smithr4rlangchain.smith.evaluationrr5r6r^ri getLoggerr9rrcrrbr-r8rArwrrrrrrrrrrrr r rrr&r1r6r8r;r[r] dataclassrarrrrr_RUN_ON_DATASET_DOCSTRINGreplacer=r>r?rs"D" '   06<C8MM><=#FELL*''7 5B@   8 $ Ruh' '( dVS[    HRuh// 02C CD3y3F4F4R R R': 0: :  : z/ d Y $ P1 B"" "2" "JC CC2C C(3  " < $#0 $$. $($@@ * @  @@)@!@@"@@F 4 $ 4# 4% 4) 4 8 4= $===) = $ = % ==N!%48)-= = =  =  = 2 ='==J!%48)- !    2 'J59 3 3 3 3 2 3 - 3|!%48)-; ; ;;  ; 2 ;';;F!%48)- !    2 'J59 5 5 5 5 2 5 - 5z26 $6::5 :5:51:5 :5 / :5  :54:57:5z%    D   :,6:6:"&15!%>< ><><1>< 3 >< 4 ><>< ></><><><><>