ih,hddlZddlmZmZmZmZddlmZddlm Z ddl m Z ddl m Z ddlmZdZd Zd Zd Zd Zd ZdZdZdZdZdZdZdZdZdZdZdZ dZ!dZ"eeegZ#edgdzedgdzedgdze!dgdze"dgdzedgdzedgdzedegdzedegdzedegdze degdzi Z$d efd!Z%Gd"d#e Z&y)$N)AnyDictListOptional)UUID)BaseCallbackHandler) LLMResult guard_import) import_pandas token_usage total_tokens prompt_tokenscompletion_tokensrun_id model_namegoodbadneutralsuccessfailureiipromptresponsecontextdurationfeedback llm_statusfiddler z$123e4567-e89b-12d3-a456-426614174000returnctddS)zLImport the fiddler python package and raise an error if it is not installed.rzfiddler-client)pip_namer l/var/www/html/dev/engine/venv/lib/python3.12/site-packages/langchain_community/callbacks/fiddler_callback.pyimport_fiddlerr(8s  ,< ==r&ceZdZdedededededdf fd Zedefd Z dd ed e ed e d ede ede e ee fddfdZde ee fde ede de fdZdede ddfdZdede ddfdZxZS)FiddlerCallbackHandlerurlorgprojectmodelapi_keyr"Nc  t |t|_t |_||_||_||_||_ ||_ |j jt|_ i|_i|_i|_|jj#||||_|j|j$j'vr?t)d|jd |j$j+|j|jj.j1|jd}t3t5|j6D]}|j6|j8t:k(rK|jj<j>|j6|_ tB|j6|_"n|j6|j8tFk(s|jj<j>|j6|_ tHtJg|j6|_"|j|j$jM|jvr|j|j$jO|jvrft)d |jd |jd |j$jQ|j|jd |ji| |jjRjU|d |jjVjXtZt\t^gt:t`tbtdtfthtjg|jl} t)d|jd |jd |j$jo|j|j|j| yy#t,$r }t)d|jd|d}~wwxYw#t,$r#}t)d |jd|d|d}~wwxYw#t,$r#}t)d|jd|d|d}~wwxYw)a Initialize Fiddler callback handler. Args: url: Fiddler URL (e.g. https://demo.fiddler.ai). Make sure to include the protocol (http/https). org: Fiddler organization id project: Fiddler project name to publish events to model: Fiddler model name to publish events to api_key: Fiddler authentication token )org_id auth_tokenzadding project z.This only has to be done once.zError adding project z(:{e}. Fiddler integration will not work.Nr)max_inferred_cardinalityzadding dataset z to project train) project_id dataset_iddatasetinfozError adding dataset z: z#.Fiddler integration will not work.) dataset_infor6 model_taskfeaturestarget metadata_colscustom_featuresz adding model )r5r6model_id model_infozError adding model )8super__init__r(fdlr pdr+r,r-r.r/ DataFrame _dataset_dict_dfrun_id_promptsrun_id_responserun_id_starttime FiddlerApifiddler_clientget_project_namesprint add_project Exception DatasetInfofrom_dataframerangelencolumnsnameFEEDBACKDataTypeCATEGORY data_typeFEEDBACK_POSSIBLE_VALUESpossible_values LLM_STATUSSUCCESSFAILUREget_model_namesget_dataset_namesupload_dataset ModelInfofrom_dataset_info ModelTaskLLMPROMPTCONTEXTRESPONSERUN_ID TOTAL_TOKENS PROMPT_TOKENSCOMPLETION_TOKENS MODEL_NAMEDURATIONr> add_model) selfr+r,r-r.r/er9ir@ __class__s r'rBzFiddlerCallbackHandler.__init__?s& !#/   77$$]3576813#hh11#cg1V <J  |< ~F11  ##--#||#zz!ZZ) .S O+ +DLL>:>>  D!/ |2aSA==G D )$**Rs;99  sH-%O6 ?P"7=Q6 P?PP" Q+Q  Q Q=Q88Q=c |jjddtg|jjdtd|jjddtg|jjdtd|jjdd ttgd gd i |jjd dttg|jjddttggS)a[ Define custom features for the model to automatically enrich the data with. Here, we enable the following enrichments: - Automatic Embedding generation for prompt and response - Text Statistics such as: - Automated Readability Index - Coleman Liau Index - Dale Chall Readability Score - Difficult Words - Flesch Reading Ease - Flesch Kincaid Grade - Gunning Fog - Linsear Write Formula - PII - Personal Identifiable Information - Sentiment Analysis zPrompt Embedding embedding)rV enrichmentrUz Prompt CF)rV source_columncolumnzResponse Embeddingz Response CFzText Statisticstextstat statistics)automated_readability_indexcoleman_liau_indexdale_chall_readability_scoredifficult_wordsflesch_reading_easeflesch_kincaid_grade gunning_foglinsear_write_formula)rVrwrUconfigPIIpii Sentiment sentiment)rC Enrichmentrg TextEmbeddingri)rqs r'r>z&FiddlerCallbackHandler.custom_featuress* HH  '&   HH " " $) #  HH  )&!    HH " ""&+ #  HH  &%* #   " HH   *   HH   &*  W0 0 r&rprompt_responsesrrrtoken_usage_dictct|j|}|jjt|j|t |t t|g|zt|g|zt|g|zt|g|zi}|r3|jD] \} } t| tr| g|zn| || <" |jddkDr2|jj!|j"|j$|y|j'd} |jj)|j"|j$| dy#t*$r} t-d| dYd} ~ yd} ~ wwxYw) z+ Publish events to fiddler rr!records)orient)eventz$Error publishing events to fiddler: z. continuing...N)rTrHrDrErgrirjstrror]rnitems isinstanceintshaperLpublish_events_batchr-r.to_dict publish_eventrPrN) rqrrrrrr prompt_countdfkeyvaluedf_dictrrs r'_publish_eventsz&FiddlerCallbackHandler._publish_eventssW4..v67 WW  ++F3*V 48*|3ZL<7ZL<7     .446 V U4>uc4J5'L0PU3 V xx{Q##88tzzSUV**I*6##11LL$**GAJ2  6qcI   s 4AE8AE E" EE" serializedpromptskwargsc |t}||j|<ttjdz|j|<y)N)rjrHrtimerJ)rqrrrrs r' on_llm_startz#FiddlerCallbackHandler.on_llm_start&s=&-F#(+DIIK$,>(?f%r&rc .|j}|t}ttjdz|j|z }d}i}t |j tr`|j jD cic]\}} |tttfvr|| }}} |j jtd}|D cgc]} | jddj } } |j!|| |t"||ycc} }wcc} w)Nrr)flattenrjrrrJr llm_outputdictrrkrlrmgetrn generationstextrr^) rqrrflattened_llmresultr run_durationrrkv llmresultrs r' on_llm_endz!FiddlerCallbackHandler.on_llm_end-s&..0499;-.1F1Fv1NN   h))4 0%//557 Aq}6GHH1    ",,00R@J?R 1:I ! !! $Q ' , ,             s  D  #Derrorc |t}ttjdz|j|z }|j |dgt |j |z|ty)Nrr)rjrrrJrrTrHr_)rqrrrrs r' on_llm_errorz#FiddlerCallbackHandler.on_llm_errorIs^tyy{T)*T-B-B6-JJ  RD3t226:;;Xw r&)rN)__name__ __module__ __qualname__rrBpropertylistr>rrrrrrrrr r BaseExceptionr __classcell__)rts@r'r*r*>s:u uu u  u  u unC C C V%'59((s)( (  ( SM (#4S>2( (T@sCx.@379@HK@ @ 9   8 - 3 4 r&r*)'rtypingrrrruuidrlangchain_core.callbacksrlangchain_core.outputsr langchain_core.utilsr #langchain_community.callbacks.utilsr TOKEN_USAGErkrlrmrjrnGOODBADNEUTRALr^r_DEFAULT_MAX_TOKENDEFAULT_MAX_DURATIONrgrirhrorWr]r[rFr(r*r%r&r'rsK ,,8,-=    '                #w/ YK"  ykB i[2  vhm b  b  3 4r 91'(1,A()A-,-1 q&'!+  >> Q 0Q r&