diff --git a/src/khoj/processor/conversation/gpt4all/chat_model.py b/src/khoj/processor/conversation/gpt4all/chat_model.py
index d0cf22ff..6f91fdf4 100644
--- a/src/khoj/processor/conversation/gpt4all/chat_model.py
+++ b/src/khoj/processor/conversation/gpt4all/chat_model.py
@@ -129,10 +129,10 @@ def converse_offline(
     compiled_references_message = "\n\n".join({f"{item}" for item in references})
 
     # Get Conversation Primer appropriate to Conversation Type
-    if conversation_command == ConversationCommand.General:
-        conversation_primer = user_query
-    elif conversation_command == ConversationCommand.Notes and is_none_or_empty(compiled_references_message):
+    if conversation_command == ConversationCommand.Notes and is_none_or_empty(compiled_references_message):
         return iter([prompts.no_notes_found.format()])
+    elif conversation_command == ConversationCommand.General or is_none_or_empty(compiled_references_message):
+        conversation_primer = user_query
     else:
         conversation_primer = prompts.notes_conversation_llamav2.format(
             query=user_query, references=compiled_references_message
diff --git a/src/khoj/processor/conversation/openai/gpt.py b/src/khoj/processor/conversation/openai/gpt.py
index eef5de1e..8105c2d7 100644
--- a/src/khoj/processor/conversation/openai/gpt.py
+++ b/src/khoj/processor/conversation/openai/gpt.py
@@ -119,11 +119,11 @@ def converse(
     compiled_references = "\n\n".join({f"# {item}" for item in references})
 
     # Get Conversation Primer appropriate to Conversation Type
-    if conversation_command == ConversationCommand.General:
-        conversation_primer = prompts.general_conversation.format(current_date=current_date, query=user_query)
-    elif conversation_command == ConversationCommand.Notes and is_none_or_empty(compiled_references):
+    if conversation_command == ConversationCommand.Notes and is_none_or_empty(compiled_references):
         completion_func(chat_response=prompts.no_notes_found.format())
         return iter([prompts.no_notes_found.format()])
+    elif conversation_command == ConversationCommand.General or is_none_or_empty(compiled_references):
+        conversation_primer = prompts.general_conversation.format(current_date=current_date, query=user_query)
     else:
         conversation_primer = prompts.notes_conversation.format(
             current_date=current_date, query=user_query, references=compiled_references