diff --git a/src/khoj/processor/conversation/gpt.py b/src/khoj/processor/conversation/gpt.py
index 312f2535..c0ba1517 100644
--- a/src/khoj/processor/conversation/gpt.py
+++ b/src/khoj/processor/conversation/gpt.py
@@ -244,14 +244,28 @@ def converse(references, user_query, conversation_log={}, model="gpt-3.5-turbo",
     compiled_references = "\n\n".join({f"# {item}" for item in references})
 
     personality_primer = "You are Khoj, a friendly, smart and helpful personal assistant."
-    conversation_primer = f"""
+    conversation_primers = {
+        "general": f"""
+Using your general knowledge and our past conversations as context, answer the following question.
+Current Date: {datetime.now().strftime("%Y-%m-%d")}
+
+Question: {user_query}
+""".strip(),
+        "notes": f"""
 Using the notes and our past conversations as context, answer the following question.
 Current Date: {datetime.now().strftime("%Y-%m-%d")}
 
 Notes:
 {compiled_references}
 
-Question: {user_query}"""
+Question: {user_query}
+""".strip(),
+    }
+
+    # Get Conversation Primer appropriate to Conversation Type
+    conversation_type = "general" if user_query.startswith("@general") or compiled_references.strip() == "" else "notes"
+    logger.debug(f"Conversation Type: {conversation_type}")
+    conversation_primer = conversation_primers.get(conversation_type)
 
     # Setup Prompt with Primer or Conversation History
     messages = generate_chatml_messages_with_context(
diff --git a/tests/test_chat_actors.py b/tests/test_chat_actors.py
index cd1f9d4d..df710ed3 100644
--- a/tests/test_chat_actors.py
+++ b/tests/test_chat_actors.py
@@ -277,8 +277,8 @@ def test_answer_from_chat_history_and_currently_retrieved_content():
 
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.chatquality
-def test_no_answer_in_chat_history_or_retrieved_content():
-    "Chat actor should say don't know as not enough contexts in chat history or retrieved to answer question"
+def test_refuse_answering_unanswerable_question():
+    "Chat actor should not try make up answers to unanswerable questions."
     # Arrange
     message_list = [
         ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []),
@@ -294,7 +294,15 @@ def test_no_answer_in_chat_history_or_retrieved_content():
     )
 
     # Assert
-    expected_responses = ["don't know", "do not know", "no information", "do not have", "don't have"]
+    expected_responses = [
+        "don't know",
+        "do not know",
+        "no information",
+        "do not have",
+        "don't have",
+        "cannot answer",
+        "I'm sorry",
+    ]
     assert len(response) > 0
     assert any([expected_response in response for expected_response in expected_responses]), (
         "Expected chat actor to say they don't know in response, but got: " + response
diff --git a/tests/test_chat_director.py b/tests/test_chat_director.py
index 5feaa3f3..0d29ae53 100644
--- a/tests/test_chat_director.py
+++ b/tests/test_chat_director.py
@@ -212,7 +212,7 @@ def test_answer_general_question_not_in_chat_history_or_retrieved_content(chat_c
     populate_chat_history(message_list)
 
     # Act
-    response = chat_client.get(f'/api/chat?q=""Write a haiku about unit testing"')
+    response = chat_client.get(f'/api/chat?q=""Write a haiku about unit testing. Do not say anything else."')
     response_message = response.json()["response"]
 
     # Assert