klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-10 13:26:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make Llama 2 stop generating response on hitting specified stop words

Browse Source 
It would previously some times start generating fake dialogue with
it's internal prompt patterns of <s>[INST] in responses.

This is a jarring experience. Stop generation response when hit <s>

Resolves #398
This commit is contained in:
Debanjum Singh Solanky
2023-08-01 18:53:46 -07:00
parent aa6846395d
commit 6e4050fa81
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/khoj/processor/conversation/gpt4all/chat_model.py
View File

@@ -160,6 +160,7 @@ def llm_thread(g, messages: List[ChatMessage], model: GPT4All):
for message in conversation_history for message in conversation_history
] ]
stop_words = ["<s>"]
chat_history = "".join(formatted_messages) chat_history = "".join(formatted_messages)
templated_system_message = prompts.system_prompt_llamav2.format(message=system_message.content) templated_system_message = prompts.system_prompt_llamav2.format(message=system_message.content)
templated_user_message = prompts.general_conversation_llamav2.format(query=user_message.content) templated_user_message = prompts.general_conversation_llamav2.format(query=user_message.content)
@@ -168,6 +169,9 @@ def llm_thread(g, messages: List[ChatMessage], model: GPT4All):
state.chat_lock.acquire() state.chat_lock.acquire()
try: try:
for response in response_iterator: for response in response_iterator:
if any(stop_word in response.strip() for stop_word in stop_words):
logger.debug(f"Stop response as hit stop word in {response}")
break
g.send(response) g.send(response)
finally: finally:
state.chat_lock.release() state.chat_lock.release()