klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-05 21:29:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Read text files as utf-8, instead of default os locale

Browse Source 
On Windows, the default locale isn't utf8. Khoj had regressed to
reading files in OS specified locale encoding, e.g cp1252, cp949 etc.

It now explicitly uses utf8 encoding to read text files for indexing

Resolves #495, resolves #472
This commit is contained in:
Debanjum Singh Solanky
2023-10-17 21:31:15 -07:00
parent 3d4576ae38
commit d9d133dfb9
3 changed files with 14 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/khoj/routers/indexer.py
View File

@@ -72,7 +72,7 @@ async def update(
raise HTTPException(status_code=401, detail="Invalid API Key")
state.config_lock.acquire()
try:
logger.info(f"📬 Updating content index via API call by {client}")
logger.info(f"📬 Updating content index via API call by {client} client")
org_files: Dict[str, str] = {}
markdown_files: Dict[str, str] = {}
pdf_files: Dict[str, str] = {}
@@ -139,7 +139,8 @@ async def update(
except Exception as e:
logger.error(
f"🚨 Failed to {force} update {t} content index triggered via API call by {client}: {e}", exc_info=True
f"🚨 Failed to {force} update {t} content index triggered via API call by {client} client: {e}",
exc_info=True,
)
finally:
state.config_lock.release()
@@ -154,7 +155,7 @@ async def update(
host=host,
)
logger.info(f"📪 Content index updated via API call by {client}")
logger.info(f"📪 Content index updated via API call by {client} client")
return Response(content="OK", status_code=200)

9
src/khoj/utils/fs_syncer.py
View File

@@ -1,6 +1,5 @@
import logging
import glob
import base64
from typing import Optional
from bs4 import BeautifulSoup
@@ -69,7 +68,7 @@ def get_plaintext_files(config: TextContentConfig) -> dict[str, str]:
filename_to_content_map = {}
for file in all_target_files:
with open(file, "r") as f:
with open(file, "r", encoding="utf8") as f:
try:
plaintext_content = f.read()
if file.endswith(("html", "htm", "xml")):
@@ -115,7 +114,7 @@ def get_org_files(config: TextContentConfig):
filename_to_content_map = {}
for file in all_org_files:
with open(file, "r") as f:
with open(file, "r", encoding="utf8") as f:
try:
filename_to_content_map[file] = f.read()
except Exception as e:
@@ -137,7 +136,7 @@ def get_markdown_files(config: TextContentConfig):
logger.debug("At least one of markdown-files or markdown-file-filter is required to be specified")
return {}
"Get Markdown files to process"
# Get markdown files to process
absolute_markdown_files, filtered_markdown_files = set(), set()
if markdown_files:
absolute_markdown_files = {get_absolute_path(markdown_file) for markdown_file in markdown_files}
@@ -164,7 +163,7 @@ def get_markdown_files(config: TextContentConfig):
filename_to_content_map = {}
for file in all_markdown_files:
with open(file, "r") as f:
with open(file, "r", encoding="utf8") as f:
try:
filename_to_content_map[file] = f.read()
except Exception as e: