From 84a8088c2b1f7b9d505a9f00ddfd093520039c26 Mon Sep 17 00:00:00 2001 From: Debanjum Date: Thu, 7 Nov 2024 15:23:30 -0800 Subject: [PATCH] Only evaluate non-empty responses to reduce eval script latency, cost Empty responses by Khoj will always be an incorrect response, so no need to make call to an evaluator agent to check that --- tests/eval_frames.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/eval_frames.py b/tests/eval_frames.py index 8a1d849e..ae44f63e 100644 --- a/tests/eval_frames.py +++ b/tests/eval_frames.py @@ -113,7 +113,11 @@ def process_batch(batch, counter, results, dataset_length): agent_response = get_agent_response(prompt) # Evaluate response - evaluation = evaluate_response(prompt, agent_response, answer) + if agent_response is None or agent_response.strip() == "": + evaluation["decision"] = False + evaluation["explanation"] = "Agent response is empty. This maybe due to a service error." + else: + evaluation = evaluate_response(prompt, agent_response, answer) # Store results results.append(