klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-02 21:19:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Only show confusing fallback tokenizer used logs in high verbosity mode

Browse Source
This commit is contained in:
Debanjum
2025-01-25 18:34:30 +07:00
parent aebdd174ec
commit af49375884
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/khoj/processor/conversation/utils.py
View File

@@ -534,9 +534,10 @@ def truncate_messages(
encoder = download_model(model_name).tokenizer()
except:
encoder = tiktoken.encoding_for_model(default_tokenizer)
logger.debug(
f"Fallback to default chat model tokenizer: {default_tokenizer}.\nConfigure tokenizer for model: {model_name} in Khoj settings to improve context stuffing."
)
if state.verbose > 2:
logger.debug(
f"Fallback to default chat model tokenizer: {default_tokenizer}.\nConfigure tokenizer for model: {model_name} in Khoj settings to improve context stuffing."
)
# Extract system message from messages
system_message = None