Split Configure API into Content, Model API paths (#857)

## Major: Breaking Changes
- Move API endpoints under /configure/<type>/model to /api/model/<type>
- Move API endpoints under /api/configure/content/ to /api/content/
- Accept file deletion requests by clients during sync
- Split /api/v1/index/update into /api/content PUT, PATCH API endpoints

## Minor: Create New API Endpoint
- Create API endpoints to get user content configurations

Related: #852

src/interface/desktop/main.js

@@ -233,11 +233,15 @@ function pushDataToKhoj (regenerate = false) {
 
     // Request indexing files on server. With upto 1000 files in each request
     for (let i = 0; i < filesDataToPush.length; i += 1000) {
+        const syncUrl = `${hostURL}/api/content?client=desktop`;
         const filesDataGroup = filesDataToPush.slice(i, i + 1000);
         const formData = new FormData();
         filesDataGroup.forEach(fileData => { formData.append('files', fileData.blob, fileData.path) });
-        let request = axios.post(`${hostURL}/api/v1/index/update?force=${regenerate}&client=desktop`, formData, { headers });
-        requests.push(request);
+        requests.push(
+            regenerate
+            ? axios.put(syncUrl, formData, { headers })
+            : axios.patch(syncUrl, formData, { headers })
+        );
     }
 
     // Wait for requests batch to finish

src/interface/desktop/search.html

@@ -212,7 +212,7 @@
             const headers = { 'Authorization': `Bearer ${khojToken}` };
 
             // Populate type dropdown field with enabled content types only
-            fetch(`${hostURL}/api/configure/types`, { headers })
+            fetch(`${hostURL}/api/content/types`, { headers })
                 .then(response => response.json())
                 .then(enabled_types => {
                     // Show warning if no content types are enabled

src/interface/emacs/khoj.el

@@ -424,12 +424,12 @@ Auto invokes setup steps on calling main entrypoint."
   "Send multi-part form `BODY' of `CONTENT-TYPE' in request to khoj server.
 Append 'TYPE-QUERY' as query parameter in request url.
 Specify `BOUNDARY' used to separate files in request header."
-  (let ((url-request-method "POST")
+  (let ((url-request-method ((if force) "PUT" "PATCH"))
         (url-request-data body)
           (url-request-extra-headers `(("content-type" . ,(format "multipart/form-data; boundary=%s" boundary))
                                        ("Authorization" . ,(format "Bearer %s" khoj-api-key)))))
       (with-current-buffer
-          (url-retrieve (format "%s/api/v1/index/update?%s&force=%s&client=emacs" khoj-server-url type-query (or force "false"))
+          (url-retrieve (format "%s/api/content?%s&client=emacs" khoj-server-url type-query)
                         ;; render response from indexing API endpoint on server
                         (lambda (status)
                           (if (not (plist-get status :error))
@@ -697,7 +697,7 @@ Optionally apply CALLBACK with JSON parsed response and CBARGS."
 
 (defun khoj--get-enabled-content-types ()
   "Get content types enabled for search from API."
-  (khoj--call-api "/api/configure/types" "GET" nil `(lambda (item) (mapcar #'intern item))))
+  (khoj--call-api "/api/content/types" "GET" nil `(lambda (item) (mapcar #'intern item))))
 
 (defun khoj--query-search-api-and-render-results (query content-type buffer-name &optional rerank is-find-similar)
   "Query Khoj Search API with QUERY, CONTENT-TYPE and RERANK as query params.

src/interface/obsidian/src/utils.ts

@@ -89,10 +89,11 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
     for (let i = 0; i < fileData.length; i += 1000) {
         const filesGroup = fileData.slice(i, i + 1000);
         const formData = new FormData();
+        const method = regenerate ? "PUT" : "PATCH";
         filesGroup.forEach(fileItem => { formData.append('files', fileItem.blob, fileItem.path) });
         // Call Khoj backend to update index with all markdown, pdf files
-        const response = await fetch(`${setting.khojUrl}/api/v1/index/update?force=${regenerate}&client=obsidian`, {
-            method: 'POST',
+        const response = await fetch(`${setting.khojUrl}/api/content?client=obsidian`, {
+            method: method,
             headers: {
                 'Authorization': `Bearer ${setting.khojApiKey}`,
             },

src/interface/web/app/common/chatFunctions.ts

@@ -277,8 +277,8 @@ export function uploadDataForIndexing(
     // Wait for all files to be read before making the fetch request
     Promise.all(fileReadPromises)
         .then(() => {
-            return fetch("/api/v1/index/update?force=false&client=web", {
-                method: "POST",
+            return fetch("/api/content?client=web", {
+                method: "PATCH",
                 body: formData,
             });
         })

src/interface/web/app/components/modelPicker/modelPicker.tsx

@@ -68,8 +68,8 @@ interface ModelPickerProps {
 }
 
 export const ModelPicker: React.FC<any> = (props: ModelPickerProps) => {
-    const { data: models } = useOptionsRequest('/api/configure/chat/model/options');
-    const { data: selectedModel } = useSelectedModel('/api/configure/chat/model');
+    const { data: models } = useOptionsRequest('/api/model/chat/options');
+    const { data: selectedModel } = useSelectedModel('/api/model/chat');
     const [openLoginDialog, setOpenLoginDialog] = React.useState(false);
 
     let userData = useAuthenticatedData();
@@ -94,7 +94,7 @@ export const ModelPicker: React.FC<any> = (props: ModelPickerProps) => {
             props.setModelUsed(model);
         }
 
-        fetch('/api/configure/chat/model' + '?id=' + String(model.id), { method: 'POST', body: JSON.stringify(model) })
+        fetch('/api/model/chat' + '?id=' + String(model.id), { method: 'POST', body: JSON.stringify(model) })
             .then((response) => {
                 if (!response.ok) {
                     throw new Error('Failed to select model');

src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx

@@ -148,7 +148,7 @@ interface FilesMenuProps {
 
 function FilesMenu(props: FilesMenuProps) {
     // Use SWR to fetch files
-    const { data: files, error } = useSWR<string[]>(props.conversationId ? '/api/configure/content/computer' : null, fetcher);
+    const { data: files, error } = useSWR<string[]>(props.conversationId ? '/api/content/computer' : null, fetcher);
     const { data: selectedFiles, error: selectedFilesError } = useSWR(props.conversationId ? `/api/chat/conversation/file-filters/${props.conversationId}` : null, fetcher);
     const [isOpen, setIsOpen] = useState(false);
     const [unfilteredFiles, setUnfilteredFiles] = useState<string[]>([]);