klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-09 13:25:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Handle offline chat model retrieval when no internet

Browse Source 
Offline chat shouldn't fail on retrieve_model when no internet,
if model was previously downloaded and usable offline
This commit is contained in:
Debanjum Singh Solanky
2023-12-04 13:46:25 -05:00
parent 2b09caa237
commit 316b7d471a
1 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/khoj/processor/conversation/offline/utils.py
View File

@@ -12,11 +12,12 @@ def download_model(model_name: str):
logger.info("There was an error importing GPT4All. Please run pip install gpt4all in order to install it.") logger.info("There was an error importing GPT4All. Please run pip install gpt4all in order to install it.")
raise e raise e
# Download the chat model # Decide whether to load model to GPU or CPU
chat_model_config = None
try:
# Download the chat model and its config
chat_model_config = gpt4all.GPT4All.retrieve_model(model_name=model_name, allow_download=True) chat_model_config = gpt4all.GPT4All.retrieve_model(model_name=model_name, allow_download=True)
# Decide whether to load model to GPU or CPU
try:
# Try load chat model to GPU if: # Try load chat model to GPU if:
# 1. Loading chat model to GPU isn't disabled via CLI and # 1. Loading chat model to GPU isn't disabled via CLI and
# 2. Machine has GPU # 2. Machine has GPU
@@ -26,6 +27,12 @@ def download_model(model_name: str):
) )
except ValueError: except ValueError:
device = "cpu" device = "cpu"
except Exception as e:
if chat_model_config is None:
device = "cpu" # Fallback to CPU as can't determine if GPU has enough memory
logger.debug(f"Unable to download model config from gpt4all website: {e}")
else:
raise e
# Now load the downloaded chat model onto appropriate device # Now load the downloaded chat model onto appropriate device
chat_model = gpt4all.GPT4All(model_name=model_name, device=device, allow_download=False) chat_model = gpt4all.GPT4All(model_name=model_name, device=device, allow_download=False)