diff --git a/src/khoj/search_type/image_search.py b/src/khoj/search_type/image_search.py
index 8c0a3cdb..fd5107ad 100644
--- a/src/khoj/search_type/image_search.py
+++ b/src/khoj/search_type/image_search.py
@@ -12,7 +12,6 @@ from sentence_transformers import SentenceTransformer, util
 from PIL import Image
 from tqdm import trange
 import torch
-from khoj.utils import state
 
 # Internal Packages
 from khoj.utils.helpers import get_absolute_path, get_from_dict, resolve_absolute_path, load_model, timer
@@ -26,9 +25,6 @@ logger = logging.getLogger(__name__)
 
 
 def initialize_model(search_config: ImageSearchConfig):
-    # Initialize Model
-    torch.set_num_threads(4)
-
     # Convert model directory to absolute path
     search_config.model_directory = resolve_absolute_path(search_config.model_directory)
 
diff --git a/src/khoj/utils/rawconfig.py b/src/khoj/utils/rawconfig.py
index 4c97aedd..d36a36ff 100644
--- a/src/khoj/utils/rawconfig.py
+++ b/src/khoj/utils/rawconfig.py
@@ -72,6 +72,9 @@ class ImageSearchConfig(ConfigBase):
     encoder_type: Optional[str] = None
     model_directory: Optional[Path] = None
 
+    class Config:
+        protected_namespaces = ()
+
 
 class SearchConfig(ConfigBase):
     image: Optional[ImageSearchConfig] = None