klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-04 05:39:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Drop long words from compiled entries to be within max token limit of models

Browse Source 
Long words (>500 characters) provide less useful context to models.

Dropping very long words allow models to create better embeddings by
passing more of the useful context from the entry to the model
This commit is contained in:
Debanjum Singh Solanky
2023-01-07 16:59:33 -03:00
parent 6a30a13326
commit 826f9dc054
2 changed files with 22 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/processor/text_to_jsonl.py
View File

@@ -24,11 +24,13 @@ class TextToJsonl(ABC):
return lambda entry: hashlib.md5(bytes(getattr(entry, key), encoding='utf-8')).hexdigest()
@staticmethod
def split_entries_by_max_tokens(entries: list[Entry], max_tokens: int=256) -> list[Entry]:
def split_entries_by_max_tokens(entries: list[Entry], max_tokens: int=256, max_word_length: int=500) -> list[Entry]:
"Split entries if compiled entry length exceeds the max tokens supported by the ML model."
chunked_entries: list[Entry] = []
for entry in entries:
compiled_entry_words = entry.compiled.split()
# Drop long words instead of having entry truncated to maintain quality of entry processed by models
compiled_entry_words = [word for word in compiled_entry_words if len(word) <= max_word_length]
for chunk_index in range(0, len(compiled_entry_words), max_tokens):
compiled_entry_words_chunk = compiled_entry_words[chunk_index:chunk_index + max_tokens]
compiled_entry_chunk = ' '.join(compiled_entry_words_chunk)