Any idea why the FaithfulnessEvaluator returns an answer to the question instead of a Yes or No ?
import pandas as pd
def display_eval_df(query, response, eval_result):
eval_df = pd.DataFrame(
{
"Query": str(query),
"Response": str(response),
"Source": response.source_nodes[0].node.get_content()[:500] + "...",
"Evaluation Result": eval_result.feedback
},
index=[0],
)
eval_df = eval_df.style.set_properties(
**{
"inline-size": "600px",
"overflow-wrap": "break-word",
},
subset=["Response", "Source"]
)
display(eval_df)
prompt_helper = PromptHelper(context_window=4096,
num_output=256,
chunk_overlap_ratio=0.1,
chunk_size_limit=None)
service_context = ServiceContext.from_defaults(llm=llm,
embed_model=embed_model,
prompt_helper=prompt_helper)
query_engine = rag.custom_retriever(query=None,
documents=documents,
method='top_k',
rag=True,
chunk_size=512,
similarity_top_k=1,
eval=True)
faithfulness_evaluator = FaithfulnessEvaluator(service_context=service_context)
response_vector = query_engine.query(eval_questions[1])
eval_result = faithfulness_evaluator.evaluate_response(response=response_vector)
display_eval_df(eval_questions[1], response_vector, eval_result)