ih4zdZddlmZmZmZmZmZmZmZddl m Z ddl m Z ddl mZddlmZddlmZmZddlmZmZdd lmZmZmZdd lmZdd lmZdd l m!Z!m"Z"dd l#m$Z%eeeegeeee&ffZ'eeeeeegeeee&ffZ(GddeZ)Gdde)Z*ee'ee"fZ+ee!e,e)fZ-GddeZ.y)z!Configuration for run evaluators.)AnyCallableDictListOptionalSequenceUnion) Embeddings)BaseLanguageModel)BasePromptTemplate) RunEvaluator)EvaluationResultEvaluationResults)ExampleRun) BaseModel ConfigDictField) CRITERIA_TYPE)EmbeddingDistance) EvaluatorTypeStringEvaluator)StringDistancec2eZdZUdZeed<deeeffdZ y) EvalConfigaConfiguration for a given run evaluator. Parameters ---------- evaluator_type : EvaluatorType The type of evaluator to use. Methods ------- get_kwargs() Get the keyword arguments for the evaluator configuration. evaluator_typereturnc:i}|D]\}}|dk(r ||||<|S)zGet the keyword arguments for the load_evaluator call. Returns ------- Dict[str, Any] The keyword arguments for the load_evaluator call. r)selfkwargsfieldvals _/var/www/html/dev/engine/venv/lib/python3.12/site-packages/langchain/smith/evaluation/config.py get_kwargszEvalConfig.get_kwargs0s@ JE3((F5M   N) __name__ __module__ __qualname____doc__r__annotations__rstrrr%rr&r$rrs# "!DcNr&rcveZdZUdZdZeeed< dZeeed< dZ eeed< de ee fffd Z xZ S)SingleKeyEvalConfigzBConfiguration for a run evaluator that only requires a single key.N reference_keyprediction_key input_keyrcVt|}dD]}|j|d|S)N)r/r0r1)superr%pop)r r!key __class__s r$r%zSingleKeyEvalConfig.get_kwargsQs2#%C "C JJsD ! " r&)r'r(r)r*r/rr,r+r0r1rrr% __classcell__r6s@r$r.r.Cs\L#'M8C='@$(NHSM( $Ix}#BDcNr&r.ceZdZUdZeeZeee e fe d< dZ e ee e d< dZe eee d< dZe ee d< dZe ee d< dZe ee d < dZe ee d < ed ZGd deZGddeZGddeZGddeZGddeZGddeZGddeZGddeZ Gdde!Z"Gdd eZ#Gd!d"eZ$Gd#d$eZ%Gd%d&e%Z&y)' RunEvalConfiga0Configuration for a run evaluation. Parameters ---------- evaluators : List[Union[EvaluatorType, EvalConfig, RunEvaluator, Callable]] Configurations for which evaluators to apply to the dataset run. Each can be the string of an :class:`EvaluatorType `, such as EvaluatorType.QA, the evaluator type string ("qa"), or a configuration for a given evaluator (e.g., :class:`RunEvalConfig.QA `). custom_evaluators : Optional[List[Union[RunEvaluator, StringEvaluator]]] Custom evaluators to apply to the dataset run. reference_key : Optional[str] The key in the dataset run to use as the reference string. If not provided, it will be inferred automatically. prediction_key : Optional[str] The key from the traced run's outputs dictionary to use to represent the prediction. If not provided, it will be inferred automatically. input_key : Optional[str] The key from the traced run's inputs dictionary to use to represent the input. If not provided, it will be inferred automatically. eval_llm : Optional[BaseLanguageModel] The language model to pass to any evaluators that use a language model. )default_factory evaluatorsNcustom_evaluatorsbatch_evaluatorsr/r0r1eval_llmTarbitrary_types_allowedceZdZUdZdZeeed<dZee ed<e jZ e ed< d deede ddffd ZxZS) RunEvalConfig.Criteriaa"Configuration for a reference-free criteria evaluator. Parameters ---------- criteria : Optional[CRITERIA_TYPE] The criteria to evaluate. llm : Optional[BaseLanguageModel] The language model to use for the evaluation chain. Ncriteriallmrr!rc (t|dd|i|yNrDrr3__init__r rDr!r6s r$rIzRunEvalConfig.Criteria.__init__ G  9h 9& 9r&N)r'r(r)r*rDrrr+rEr rCRITERIArrrIr7r8s@r$CriteriarCse -1(=)0+/X' (/(5(>(> >7; :$]3 :FI :  : :r&rNceZdZUdZdZeeed<dZee ed<e jZ e ed< d deede ddffd ZxZS) RunEvalConfig.LabeledCriteriaa,Configuration for a labeled (with references) criteria evaluator. Parameters ---------- criteria : Optional[CRITERIA_TYPE] The criteria to evaluate. llm : Optional[BaseLanguageModel] The language model to use for the evaluation chain. NrDrErr!rc (t|dd|i|yrGrHrJs r$rIz&RunEvalConfig.LabeledCriteria.__init__rKr&rL)r'r(r)r*rDrrr+rEr rLABELED_CRITERIArrrIr7r8s@r$LabeledCriteriarPse -1(=)0+/X' (/(5(F(F F7; :$]3 :FI :  : :r&rScneZdZUdZej Zeed<dZe e ed<dZ e e ed<e dZy)RunEvalConfig.EmbeddingDistanceaIConfiguration for an embedding distance evaluator. Parameters ---------- embeddings : Optional[Embeddings] The embeddings to use for computing the distance. distance_metric : Optional[EmbeddingDistanceEnum] The distance metric to use for computing the distance. rN embeddingsdistance_metricTr@)r'r(r)r*rEMBEDDING_DISTANCErr+rVrr rWEmbeddingDistanceEnumr model_configrr&r$rrUsE )6(H(H H+/ HZ(/;?"78?!$( r&rcXeZdZUdZej Zeed<dZe e ed< dZ e ed<y)RunEvalConfig.StringDistancezConfiguration for a string distance evaluator. Parameters ---------- distance : Optional[StringDistanceEnum] The string distance metric to use. rNdistanceTnormalize_score) r'r(r)r*rSTRING_DISTANCErr+r]rStringDistanceEnumr^boolrr&r$rr\s@ )6(E(E E15(-.5 !%$ Nr&rc\eZdZUdZej Zeed<dZe e ed<dZ e e ed<y)RunEvalConfig.QAa-Configuration for a QA evaluator. Parameters ---------- prompt : Optional[BasePromptTemplate] The prompt template to use for generating the question. llm : Optional[BaseLanguageModel] The language model to use for the evaluation chain. rNrEprompt) r'r(r)r*rQArr+rErr rdr rr&r$rercs9 )6(8(8 8+/X' (//3+,3r&rec\eZdZUdZej Zeed<dZe e ed<dZ e e ed<y)RunEvalConfig.ContextQA<Configuration for a context-based QA evaluator. Parameters ---------- prompt : Optional[BasePromptTemplate] The prompt template to use for generating the question. llm : Optional[BaseLanguageModel] The language model to use for the evaluation chain. rNrErd r'r(r)r*r CONTEXT_QArr+rErr rdr rr&r$ ContextQArg9 )6(@(@ @+/X' (//3+,3r&rkc\eZdZUdZej Zeed<dZe e ed<dZ e e ed<y)RunEvalConfig.CoTQArhrNrErdrirr&r$CoTQArnrlr&roc4eZdZUdZej Zeed<y)RunEvalConfig.JsonValidityz\Configuration for a json validity evaluator. Parameters ---------- rN)r'r(r)r*r JSON_VALIDITYrr+rr&r$ JsonValidityrq& )6(C(C Cr&rsc4eZdZUdZej Zeed<y)#RunEvalConfig.JsonEqualityEvaluatorz\Configuration for a json equality evaluator. Parameters ---------- rN)r'r(r)r*r JSON_EQUALITYrr+rr&r$JsonEqualityEvaluatorrv/rtr&rxc^eZdZUdZej Zeed<dZe ed<dZ e ed<dZ e ed<y)RunEvalConfig.ExactMatchaxConfiguration for an exact match string evaluator. Parameters ---------- ignore_case : bool Whether to ignore case when comparing strings. ignore_punctuation : bool Whether to ignore punctuation when comparing strings. ignore_numbers : bool Whether to ignore numbers when comparing strings. rF ignore_caseignore_punctuationignore_numbersN) r'r(r)r*r EXACT_MATCHrr+r{rar|r}rr&r$ ExactMatchrz8s8 )6(A(A A! T!#(D($$r&rcBeZdZUdZej Zeed<dZe ed<y)RunEvalConfig.RegexMatchzConfiguration for a regex match string evaluator. Parameters ---------- flags : int The flags to pass to the regex. Example: re.IGNORECASE. rrflagsN) r'r(r)r*r REGEX_MATCHrr+rintrr&r$ RegexMatchrJs" )6(A(A Asr&rc eZdZUdZej Zeed<dZe e ed<dZ e e ed<dZ e eed<dZe eed< d de e de eded dffd ZxZS) RunEvalConfig.ScoreStringaConfiguration for a score string evaluator. This is like the criteria evaluator but it is configured by default to return a score on the scale from 1-10. It is recommended to normalize these scores by setting `normalize_by` to 10. Parameters ---------- criteria : Optional[CRITERIA_TYPE] The criteria to evaluate. llm : Optional[BaseLanguageModel] The language model to use for the evaluation chain. normalize_by: Optional[int] = None If you want to normalize the score, the denominator to use. If not provided, the score will be between 1 and 10 (by default). prompt : Optional[BasePromptTemplate] rNrDrE normalize_byrdr!rc *t|d||d|y)N)rDrrrH)r rDrr!r6s r$rIz"RunEvalConfig.ScoreString.__init__qs G  Th\ TV Tr&)NN)r'r(r)r*r SCORE_STRINGrr+rDrrrEr rfloatrdr rrIr7r8s@r$ ScoreStringrVs ()6(B(B B,0(=)0+/X' (/(, huo,/3+,315,0 U}- U#5/ U  U   U Ur&rc0eZdZUejZeed<y) RunEvalConfig.LabeledScoreStringrN)r'r(r)rLABELED_SCORE_STRINGrr+rr&r$LabeledScoreStringrys(5(J(J Jr&r)'r'r(r)r*rlistr<rr SINGLE_EVAL_CONFIG_TYPECUSTOM_EVALUATOR_TYPEr+r=rr>BATCH_EVALUATOR_LIKEr/r,r0r1r?r rrZr.rNrSrrrerkrorsrrxrrrrrr&r$r:r:]sF d#  # ! # $ X@Dx%: ;<C8=Aht$89:A $(M8C='@$(NHSM( $Ix}#B,0Hh()0H $L:&:*:-:( / *N,N, 4 44'4 4#4 D*DD D%(%$ ( !U)!UFK[Kr&r:N)/r*typingrrrrrrr langchain_core.embeddingsr langchain_core.language_modelsr langchain_core.promptsr langsmithr langsmith.evaluation.evaluatorrrlangsmith.schemasrrpydanticrrr(langchain.evaluation.criteria.eval_chainr,langchain.evaluation.embedding_distance.baserrYlangchain.evaluation.schemarr)langchain.evaluation.string_distance.baserr`dictRUN_EVALUATOR_LIKErrr.rr,rr:rr&r$rs'GGG0<5"N*11BG(7 e$46G$MNN  c]HXg./0 -t 346 !!H*,0,OP sJ >?]KI]Kr&