Find answers from the community

Updated 11 months ago

Any idea why the FaithfulnessEvaluator

Any idea why the FaithfulnessEvaluator returns an answer to the question instead of a Yes or No ?

Plain Text
import pandas as pd

def display_eval_df(query, response, eval_result):
    eval_df = pd.DataFrame(
        {
            "Query": str(query),
            "Response": str(response),
            "Source": response.source_nodes[0].node.get_content()[:500] + "...",
            "Evaluation Result": eval_result.feedback
        },
        index=[0],
    )
    eval_df = eval_df.style.set_properties(
        **{
            "inline-size": "600px",
            "overflow-wrap": "break-word",
        },
        subset=["Response", "Source"]
    )
    display(eval_df)

prompt_helper = PromptHelper(context_window=4096, 
                            num_output=256, 
                            chunk_overlap_ratio=0.1, 
                            chunk_size_limit=None)

service_context = ServiceContext.from_defaults(llm=llm,
                                                embed_model=embed_model, 
                                                prompt_helper=prompt_helper)
query_engine = rag.custom_retriever(query=None, 
                                    documents=documents,
                                    method='top_k',
                                    rag=True,
                                    chunk_size=512,
                                    similarity_top_k=1,
                                    eval=True)

faithfulness_evaluator = FaithfulnessEvaluator(service_context=service_context)
response_vector = query_engine.query(eval_questions[1])
eval_result = faithfulness_evaluator.evaluate_response(response=response_vector)

display_eval_df(eval_questions[1], response_vector, eval_result)
Attachment
image.png
W
b
3 comments
You are passing eval_result.feedback in Evaluation result , For pass/fail you will need to change it to
"Evaluation Result":"Pass" if eval_result.passing else "Fail",
Add a reply
Sign up and join the conversation on Discord