klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-09 13:25:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve online search and allow server to skip auto webpage read

Browse Source
This commit is contained in:
sabaimran
2025-03-31 13:52:48 -05:00
parent 177560655d
commit d53b740197
1 changed files with 5 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/khoj/processor/tools/online_search.py
View File

@@ -33,6 +33,7 @@ logger = logging.getLogger(__name__)
GOOGLE_SEARCH_API_KEY = os.getenv("GOOGLE_SEARCH_API_KEY") GOOGLE_SEARCH_API_KEY = os.getenv("GOOGLE_SEARCH_API_KEY")
GOOGLE_SEARCH_ENGINE_ID = os.getenv("GOOGLE_SEARCH_ENGINE_ID") GOOGLE_SEARCH_ENGINE_ID = os.getenv("GOOGLE_SEARCH_ENGINE_ID")
SERPER_DEV_API_KEY = os.getenv("SERPER_DEV_API_KEY") SERPER_DEV_API_KEY = os.getenv("SERPER_DEV_API_KEY")
AUTO_READ_WEBPAGE = is_env_var_true("AUTO_READ_WEBPAGE")
SERPER_DEV_URL = "https://google.serper.dev/search" SERPER_DEV_URL = "https://google.serper.dev/search"
JINA_SEARCH_API_URL = "https://s.jina.ai/" JINA_SEARCH_API_URL = "https://s.jina.ai/"
@@ -131,6 +132,10 @@ async def search_online(
logger.error(f"Error searching with {search_engine}: {e}") logger.error(f"Error searching with {search_engine}: {e}")
response_dict = {} response_dict = {}
if not AUTO_READ_WEBPAGE:
yield response_dict
return
# Gather distinct web pages from organic results for subqueries without an instant answer. # Gather distinct web pages from organic results for subqueries without an instant answer.
webpages: Dict[str, Dict] = {} webpages: Dict[str, Dict] = {}
for subquery in response_dict: for subquery in response_dict: