klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-06 13:22:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update handling of images when included in the chat history with assistant message

Browse Source
This commit is contained in:
sabaimran
2024-12-08 21:46:04 -08:00
parent b660c494bc
commit eb36492ba5
1 changed files with 13 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/khoj/processor/conversation/utils.py
View File

@@ -433,20 +433,19 @@ def generate_chatml_messages_with_context(
reconstructed_context_message = ChatMessage(content=message_context, role="user") reconstructed_context_message = ChatMessage(content=message_context, role="user")
chatml_messages.insert(0, reconstructed_context_message) chatml_messages.insert(0, reconstructed_context_message)
if chat.get("images"): if chat.get("images") and role == "assistant":
if role == "assistant": # Issue: the assistant role cannot accept an image as a message content, so send it in a separate user message.
# Issue: the assistant role cannot accept an image as a message content, so send it in a separate user message. file_attachment_message = construct_structured_message(
file_attachment_message = construct_structured_message( message=prompts.generated_image_attachment.format(),
message=prompts.generated_image_attachment.format(), images=chat.get("images"),
images=chat.get("images"), model_type=model_type,
model_type=model_type, vision_enabled=vision_enabled,
vision_enabled=vision_enabled, )
) chatml_messages.append(ChatMessage(content=file_attachment_message, role="user"))
chatml_messages.append(ChatMessage(content=file_attachment_message, role="user"))
else: message_content = construct_structured_message(
message_content = construct_structured_message( chat_message, chat.get("images") if role == "user" else [], model_type, vision_enabled
chat_message, chat.get("images"), model_type, vision_enabled )
)
reconstructed_message = ChatMessage(content=message_content, role=role) reconstructed_message = ChatMessage(content=message_content, role=role)
chatml_messages.insert(0, reconstructed_message) chatml_messages.insert(0, reconstructed_message)