Log clients calling API endpoints on Khoj server

- Make API endpoints on Khoj server accept `client` as request parameter
  - Khoj API endpoints: /chat, /search, /update
- Make Khoj clients set `client` request param when calling the API endpoints on the Khoj server
  - Khoj clients: Emacs, Obsidian and Web
- Also log khoj server_version running to telemetry server

src/interface/emacs/khoj.el

@@ -634,7 +634,7 @@ CONFIG is json obtained from Khoj config API."
       (buffer-string)))
   ;; Update index on khoj server after configuration update
   (let ((khoj--server-ready? nil))
-    (url-retrieve (format "%s/api/update?t=org" khoj-server-url) #'identity)))
+    (url-retrieve (format "%s/api/update?t=org&client=emacs" khoj-server-url) #'identity)))
 
 (defun khoj--get-enabled-content-types ()
   "Get content types enabled for search from API."
@@ -651,7 +651,7 @@ CONFIG is json obtained from Khoj config API."
 Use QUERY, CONTENT-TYPE and (optional) RERANK as query params"
   (let ((rerank (or rerank "false"))
         (encoded-query (url-hexify-string query)))
-    (format "%s/api/search?q=%s&t=%s&r=%s&n=%s" khoj-server-url encoded-query content-type rerank khoj-results-count)))
+    (format "%s/api/search?q=%s&t=%s&r=%s&n=%s&client=emacs" khoj-server-url encoded-query content-type rerank khoj-results-count)))
 
 (defun khoj--query-search-api-and-render-results (query-url content-type query buffer-name)
   "Query Khoj Search with QUERY-URL.
@@ -788,7 +788,7 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
   "Send QUERY to Khoj Chat API."
   (let* ((url-request-method "GET")
          (encoded-query (url-hexify-string query))
-         (query-url (format "%s/api/chat?q=%s" khoj-server-url encoded-query)))
+         (query-url (format "%s/api/chat?q=%s&client=emacs" khoj-server-url encoded-query)))
     (with-temp-buffer
       (condition-case ex
           (progn
@@ -1031,7 +1031,7 @@ Paragraph only starts at first text after blank line."
   (let* ((force-update (if (member "--force-update" args) "true" "false"))
          ;; set content type to: specified > last used > based on current buffer > default type
          (content-type (or (transient-arg-value "--content-type=" args) (khoj--buffer-name-to-content-type (buffer-name))))
-         (update-url (format "%s/api/update?t=%s&force=%s" khoj-server-url content-type force-update))
+         (update-url (format "%s/api/update?t=%s&force=%s&client=emacs" khoj-server-url content-type force-update))
          (url-request-method "GET"))
     (progn
       (setq khoj--content-type content-type)

src/interface/obsidian/src/chat_modal.ts

@@ -35,7 +35,7 @@ export class KhojChatModal extends Modal {
         contentEl.createDiv({ attr: { id: "khoj-chat-body", class: "khoj-chat-body" } });
 
         // Get conversation history from Khoj backend
-        let chatUrl = `${this.setting.khojUrl}/api/chat?`;
+        let chatUrl = `${this.setting.khojUrl}/api/chat?client=obsidian`;
         let response = await request(chatUrl);
         let chatLogs = JSON.parse(response).response;
         chatLogs.forEach((chatLog: any) => {
@@ -120,7 +120,7 @@ export class KhojChatModal extends Modal {
 
         // Get chat response from Khoj backend
         let encodedQuery = encodeURIComponent(query);
-        let chatUrl = `${this.setting.khojUrl}/api/chat?q=${encodedQuery}`;
+        let chatUrl = `${this.setting.khojUrl}/api/chat?q=${encodedQuery}&client=obsidian`;
         let response = await request(chatUrl);
         let data = JSON.parse(response);
 

src/interface/obsidian/src/search_modal.ts

@@ -89,7 +89,7 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
     async getSuggestions(query: string): Promise<SearchResult[]> {
         // Query Khoj backend for search results
         let encodedQuery = encodeURIComponent(query);
-        let searchUrl = `${this.setting.khojUrl}/api/search?q=${encodedQuery}&n=${this.setting.resultsCount}&r=${this.rerank}`;
+        let searchUrl = `${this.setting.khojUrl}/api/search?q=${encodedQuery}&n=${this.setting.resultsCount}&r=${this.rerank}&client=obsidian`;
 
         // Get search results for markdown and pdf files
         let mdResponse = await request(`${searchUrl}&t=markdown`);

src/interface/obsidian/src/settings.ts

@@ -107,8 +107,8 @@ export class KhojSettingTab extends PluginSettingTab {
                     }, 300);
                     this.plugin.registerInterval(progress_indicator);
 
-                    await request(`${this.plugin.settings.khojUrl}/api/update?t=markdown&force=true`);
+                    await request(`${this.plugin.settings.khojUrl}/api/update?t=markdown&force=true&client=obsidian`);
-                    await request(`${this.plugin.settings.khojUrl}/api/update?t=pdf&force=true`);
+                    await request(`${this.plugin.settings.khojUrl}/api/update?t=pdf&force=true&client=obsidian`);
                     new Notice('✅ Updated Khoj index.');
 
                     // Reset button once index is updated

src/khoj/interface/web/assets/config.js

@@ -46,7 +46,7 @@ regenerateButton.addEventListener("click", (event) => {
     event.preventDefault();
     regenerateButton.style.cursor = "progress";
     regenerateButton.disabled = true;
-    fetch("/api/update?force=true")
+    fetch("/api/update?force=true&client=web")
         .then(response => response.json())
         .then(data => {
             regenerateButton.style.cursor = "pointer";

src/khoj/interface/web/chat.html

@@ -62,7 +62,7 @@
             document.getElementById("chat-input").value = "";
 
             // Generate backend API URL to execute query
-            let url = `/api/chat?q=${encodeURIComponent(query)}`;
+            let url = `/api/chat?q=${encodeURIComponent(query)}&client=web`;
 
             // Call specified Khoj API
             fetch(url)
@@ -82,7 +82,7 @@
         }
 
         window.onload = function () {
-            fetch('/api/chat')
+            fetch('/api/chat?client=web')
                 .then(response => response.json())
                 .then(data => data.response)
                 .then(chat_logs => {

src/khoj/interface/web/index.html

@@ -90,8 +90,8 @@
 
             // Generate Backend API URL to execute Search
             url = type === "image"
-                ? `/api/search?q=${encodeURIComponent(query)}&t=${type}&n=${results_count}`
+                ? `/api/search?q=${encodeURIComponent(query)}&t=${type}&n=${results_count}&client=web`
-                : `/api/search?q=${encodeURIComponent(query)}&t=${type}&n=${results_count}&r=${rerank}`;
+                : `/api/search?q=${encodeURIComponent(query)}&t=${type}&n=${results_count}&r=${rerank}&client=web`;
 
             // Execute Search and Render Results
             fetch(url)
@@ -107,7 +107,7 @@
 
         function updateIndex() {
             type = document.getElementById("type").value;
-            fetch(`/api/update?t=${type}`)
+            fetch(`/api/update?t=${type}&client=web`)
                 .then(response => response.json())
                 .then(data => {
                     console.log(data);

src/khoj/routers/api.py

@@ -70,6 +70,7 @@ def search(
     r: Optional[bool] = False,
     score_threshold: Optional[Union[float, None]] = None,
     dedupe: Optional[bool] = True,
+    client: Optional[str] = None,
 ):
     results: List[SearchResponse] = []
     if q is None or q == "":
@@ -181,14 +182,16 @@ def search(
 
     # Only log telemetry if query is new and not a continuation of previous query
     if state.previous_query is None or state.previous_query not in user_query:
-        state.telemetry += [log_telemetry(telemetry_type="api", api="search", app_config=state.config.app)]
+        state.telemetry += [
+            log_telemetry(telemetry_type="api", api="search", client=client, app_config=state.config.app)
+        ]
     state.previous_query = user_query
 
     return results
 
 
 @api.get("/update")
-def update(t: Optional[SearchType] = None, force: Optional[bool] = False):
+def update(t: Optional[SearchType] = None, force: Optional[bool] = False, client: Optional[str] = None):
     try:
         state.search_index_lock.acquire()
         state.model = configure_search(state.model, state.config, regenerate=force, t=t)
@@ -207,13 +210,13 @@ def update(t: Optional[SearchType] = None, force: Optional[bool] = False):
     else:
         logger.info("📬 Processor reconfigured via API")
 
-    state.telemetry += [log_telemetry(telemetry_type="api", api="update", app_config=state.config.app)]
+    state.telemetry += [log_telemetry(telemetry_type="api", api="update", client=client, app_config=state.config.app)]
 
     return {"status": "ok", "message": "khoj reloaded"}
 
 
 @api.get("/chat")
-def chat(q: Optional[str] = None):
+def chat(q: Optional[str] = None, client: Optional[str] = None):
     if (
         state.processor_config is None
         or state.processor_config.conversation is None
@@ -277,6 +280,6 @@ def chat(q: Optional[str] = None):
         conversation_log=meta_log.get("chat", []),
     )
 
-    state.telemetry += [log_telemetry(telemetry_type="api", api="chat", app_config=state.config.app)]
+    state.telemetry += [log_telemetry(telemetry_type="api", api="chat", client=client, app_config=state.config.app)]
 
     return {"status": status, "response": gpt_response, "context": compiled_references}

src/khoj/utils/helpers.py

@@ -3,11 +3,11 @@ from __future__ import annotations  # to avoid quoting type hints
 from collections import OrderedDict
 import datetime
 from importlib import import_module
+from importlib.metadata import version
 import logging
 from os import path
 from pathlib import Path
 import platform
-import requests
 import sys
 from time import perf_counter
 import torch
@@ -184,6 +184,7 @@ def log_telemetry(telemetry_type: str, api: str = None, client: str = None, app_
     request_body = {
         "telemetry_type": telemetry_type,
         "server_id": get_server_id(),
+        "server_version": version("khoj-assistant"),
         "os": platform.system(),
         "timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
     }

src/telemetry/telemetry.py

@@ -53,7 +53,7 @@ def v1_telemetry(telemetry_data: List[Dict[str, str]]):
         # Log telemetry data
         for item in telemetry_data:
             cur.execute(
-                "INSERT INTO usage (time, type, server_id, os, api, client) VALUES (?, ?, ?, ?, ?, ?)",
+                "INSERT INTO usage (time, type, server_id, os, api, client, server_version) VALUES (?, ?, ?, ?, ?, ?, ?)",
                 (
                     item["timestamp"],
                     item["telemetry_type"],
@@ -61,6 +61,7 @@ def v1_telemetry(telemetry_data: List[Dict[str, str]]):
                     item["os"],
                     item.get("api"),
                     item.get("client"),
+                    item.get("server_version", None),
                 ),
             )
         # Commit the changes