klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-09 13:25:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Only show confusing fallback tokenizer used logs in high verbosity mode

Browse Source
This commit is contained in:
Debanjum
2025-01-25 18:34:30 +07:00
parent aebdd174ec
commit af49375884
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/khoj/processor/conversation/utils.py
View File

@@ -534,6 +534,7 @@ def truncate_messages(
encoder = download_model(model_name).tokenizer() encoder = download_model(model_name).tokenizer()
except: except:
encoder = tiktoken.encoding_for_model(default_tokenizer) encoder = tiktoken.encoding_for_model(default_tokenizer)
if state.verbose > 2:
logger.debug( logger.debug(
f"Fallback to default chat model tokenizer: {default_tokenizer}.\nConfigure tokenizer for model: {model_name} in Khoj settings to improve context stuffing." f"Fallback to default chat model tokenizer: {default_tokenizer}.\nConfigure tokenizer for model: {model_name} in Khoj settings to improve context stuffing."
) )