klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-09 13:25:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove inline base64 images from webpages read with Firecrawl

Browse Source
This commit is contained in:
Debanjum
2025-05-02 14:06:24 -06:00
parent 559b323475
commit 5b5efe463d
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/khoj/processor/tools/online_search.py
View File

@@ -499,7 +499,7 @@ async def read_webpage_with_jina(web_url: str, api_key: str, api_url: str) -> st
async def read_webpage_with_firecrawl(web_url: str, api_key: str, api_url: str) -> str: async def read_webpage_with_firecrawl(web_url: str, api_key: str, api_url: str) -> str:
firecrawl_api_url = f"{api_url}/v1/scrape" firecrawl_api_url = f"{api_url}/v1/scrape"
headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"} headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"}
params = {"url": web_url, "formats": ["markdown"], "excludeTags": ["script", ".ad"]} params = {"url": web_url, "formats": ["markdown"], "excludeTags": ["script", ".ad"], "removeBase64Images": True}
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
async with session.post(firecrawl_api_url, json=params, headers=headers) as response: async with session.post(firecrawl_api_url, json=params, headers=headers) as response: