klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-09 21:29:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Only show confusing fallback tokenizer used logs in high verbosity mode

Browse Source
This commit is contained in:
Debanjum
2025-01-25 18:34:30 +07:00
parent aebdd174ec
commit af49375884
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/khoj/processor/conversation/utils.py
View File

@@ -534,9 +534,10 @@ def truncate_messages(
encoder = download_model(model_name).tokenizer() encoder = download_model(model_name).tokenizer()
except: except:
encoder = tiktoken.encoding_for_model(default_tokenizer) encoder = tiktoken.encoding_for_model(default_tokenizer)
logger.debug( if state.verbose > 2:
f"Fallback to default chat model tokenizer: {default_tokenizer}.\nConfigure tokenizer for model: {model_name} in Khoj settings to improve context stuffing." logger.debug(
) f"Fallback to default chat model tokenizer: {default_tokenizer}.\nConfigure tokenizer for model: {model_name} in Khoj settings to improve context stuffing."
)
# Extract system message from messages # Extract system message from messages
system_message = None system_message = None