From e0352cd8e1f294dfe011e3dbad2cbe4e71139c42 Mon Sep 17 00:00:00 2001 From: Debanjum Date: Sat, 17 May 2025 02:36:10 -0700 Subject: [PATCH] Handle unset ttft in metadata of failed chat response. Fixes evals. This was causing evals to stop processing rest of batch as well. --- src/khoj/routers/api_chat.py | 8 ++++---- tests/evals/eval.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/khoj/routers/api_chat.py b/src/khoj/routers/api_chat.py index 163ffc46..27db70f5 100644 --- a/src/khoj/routers/api_chat.py +++ b/src/khoj/routers/api_chat.py @@ -809,11 +809,11 @@ async def chat( chat_metadata = chat_metadata or {} chat_metadata["conversation_command"] = cmd_set chat_metadata["agent"] = conversation.agent.slug if conversation and conversation.agent else None - chat_metadata["latency"] = f"{latency:.3f}" - chat_metadata["ttft_latency"] = f"{ttft:.3f}" chat_metadata["cost"] = f"{cost:.5f}" - - logger.info(f"Chat response time to first token: {ttft:.3f} seconds") + chat_metadata["latency"] = f"{latency:.3f}" + if ttft: + chat_metadata["ttft_latency"] = f"{ttft:.3f}" + logger.info(f"Chat response time to first token: {ttft:.3f} seconds") logger.info(f"Chat response total time: {latency:.3f} seconds") logger.info(f"Chat response cost: ${cost:.5f}") update_telemetry_state( diff --git a/tests/evals/eval.py b/tests/evals/eval.py index f968c0a3..45e87f74 100644 --- a/tests/evals/eval.py +++ b/tests/evals/eval.py @@ -516,6 +516,7 @@ def process_batch(batch, batch_start, results, dataset_length, response_evaluato if is_none_or_empty(agent_response): decision = None explanation = "Agent response is empty. This maybe due to a service error." + eval_cost = 0.0 else: decision, explanation, eval_cost = response_evaluator(prompt, agent_response, answer, agent_references)