klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-06 13:22:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fallback to enabled, lower priority online search providers on error

Browse Source 
Make serper.dev higher priority than official google serp api because
it provides more detailed results with knowledge cards etc.
This commit is contained in:
Debanjum
2025-03-31 17:03:31 +05:30
parent 9b7442f28f
commit 3939e995e4
1 changed files with 22 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/khoj/processor/tools/online_search.py
View File

@@ -95,29 +95,37 @@ async def search_online(
yield response_dict yield response_dict
return return
search_engines = []
if SERPER_DEV_API_KEY:
search_engine = "Serper"
search_engines.append((search_engine, search_with_serper))
if GOOGLE_SEARCH_API_KEY and GOOGLE_SEARCH_ENGINE_ID: if GOOGLE_SEARCH_API_KEY and GOOGLE_SEARCH_ENGINE_ID:
search_engine = "Google" search_engine = "Google"
search_func = search_with_google search_engines.append((search_engine, search_with_google))
elif SERPER_DEV_API_KEY: if JINA_API_KEY:
search_engine = "Serper"
search_func = search_with_serper
elif JINA_API_KEY:
search_engine = "Jina" search_engine = "Jina"
search_func = search_with_jina search_engines.append((search_engine, search_with_jina))
else: search_engine = "Searxng"
search_engine = "Searxng" search_engines.append((search_engine, search_with_searxng))
search_func = search_with_searxng
logger.info(f"🌐 Searching the Internet with {search_engine} for {subqueries}") logger.info(f"🌐 Searching the Internet for {subqueries}")
if send_status_func: if send_status_func:
subqueries_str = "\n- " + "\n- ".join(subqueries) subqueries_str = "\n- " + "\n- ".join(subqueries)
async for event in send_status_func(f"**Searching the Internet for**: {subqueries_str}"): async for event in send_status_func(f"**Searching the Internet for**: {subqueries_str}"):
yield {ChatEvent.STATUS: event} yield {ChatEvent.STATUS: event}
with timer(f"Internet searches for {subqueries} took", logger): response_dict = {}
search_tasks = [search_func(subquery, location) for subquery in subqueries] for search_engine, search_func in search_engines:
search_results = await asyncio.gather(*search_tasks) with timer(f"Internet searches with {search_engine} for {subqueries} took", logger):
response_dict = {subquery: search_result for subquery, search_result in search_results} try:
search_tasks = [search_func(subquery, location) for subquery in subqueries]
search_results = await asyncio.gather(*search_tasks)
response_dict = {subquery: search_result for subquery, search_result in search_results if search_result}
if not is_none_or_empty(response_dict):
break
except Exception as e:
logger.error(f"Error searching with {search_engine}: {e}")
response_dict = {}
# Gather distinct web pages from organic results for subqueries without an instant answer. # Gather distinct web pages from organic results for subqueries without an instant answer.
webpages: Dict[str, Dict] = {} webpages: Dict[str, Dict] = {}