Update the config UI to show all files indexed with option to delete

- Given the separation of the client and server now, the web UI will no longer support configuration of local file paths of data to index
- Expose a way to show all the files that are currently set for indexing, along with an option to delete all or specific files

src/database/adapters/__init__.py

@@ -291,8 +291,11 @@ class EntryAdapters:
         return deleted_count
 
     @staticmethod
-    def delete_all_entries(user: KhojUser, file_type: str):
-        deleted_count, _ = Entry.objects.filter(user=user, file_type=file_type).delete()
+    def delete_all_entries(user: KhojUser, file_type: str = None):
+        if file_type is None:
+            deleted_count, _ = Entry.objects.filter(user=user).delete()
+        else:
+            deleted_count, _ = Entry.objects.filter(user=user, file_type=file_type).delete()
         return deleted_count
 
     @staticmethod
@@ -314,6 +317,18 @@ class EntryAdapters:
     async def user_has_entries(user: KhojUser):
         return await Entry.objects.filter(user=user).aexists()
 
+    @staticmethod
+    async def adelete_entry_by_file(user: KhojUser, file_path: str):
+        return await Entry.objects.filter(user=user, file_path=file_path).adelete()
+
+    @staticmethod
+    def aget_all_filenames(user: KhojUser):
+        return Entry.objects.filter(user=user).distinct("file_path").values_list("file_path", flat=True)
+
+    @staticmethod
+    async def adelete_all_entries(user: KhojUser):
+        return await Entry.objects.filter(user=user).adelete()
+
     @staticmethod
     def apply_filters(user: KhojUser, query: str, file_type_filter: str = None):
         q_filter_terms = Q()

src/khoj/interface/web/base_config.html

@@ -53,10 +53,10 @@
             justify-self: center;
         }
 
-        .api-settings {
+        div.section-manage-files,
+        div.api-settings {
             display: grid;
             grid-template-columns: 1fr;
-            grid-template-rows: 1fr 1fr auto;
             justify-items: start;
             gap: 8px;
             padding: 24px 24px;
@@ -64,13 +64,23 @@
             border: 1px solid rgb(229, 229, 229);
             border-radius: 4px;
             box-shadow: 0px 1px 3px 0px rgba(0,0,0,0.1),0px 1px 2px -1px rgba(0,0,0,0.8);
-         }
-         #api-settings-card-description {
+        }
+
+        div.section-manage-files {
+            width: 640px;
+        }
+
+        div.api-settings {
+            grid-template-rows: 1fr 1fr auto;
+        }
+
+        #api-settings-card-description {
             margin: 8px 0 0 0;
-         }
-         #api-settings-keys-table {
-            margin-bottom: 16px;
-         }
+        }
+
+        #api-settings-keys-table {
+        margin-bottom: 16px;
+        }
 
         div.instructions {
             font-size: large;
@@ -184,6 +194,37 @@
             text-align: left;
         }
 
+        button.remove-file-button:hover {
+            background-color: rgb(255 235 235);
+            border-radius: 3px;
+            border: none;
+            color: var(--flower);
+            padding: 4px;
+            cursor: pointer;
+        }
+
+        button.remove-file-button {
+            background-color: rgb(253 214 214);
+            border-radius: 3px;
+            border: none;
+            color: var(--flower);
+            padding: 4px;
+        }
+
+        div.file-element {
+            display: grid;
+            grid-template-columns: 1fr auto;
+            border: 1px solid rgb(229, 229, 229);
+            border-radius: 4px;
+            box-shadow: 0px 1px 3px 0px rgba(0,0,0,0.1),0px 1px 2px -1px rgba(0,0,0,0.8);
+            padding: 4px;
+            margin-bottom: 8px;
+        }
+
+        div.remove-button-container {
+            text-align: right;
+        }
+
         button.card-button.happy {
             color: var(--leaf);
         }
@@ -246,6 +287,11 @@
             cursor: pointer;
         }
 
+        a {
+            color: #3b82f6;
+            text-decoration: none;
+        }
+
         @media screen and (max-width: 700px) {
             .section-cards {
                 grid-template-columns: 1fr;
@@ -255,7 +301,7 @@
             body {
                 display: grid;
                 grid-template-columns: 1fr;
-                grid-template-rows: 1fr auto auto auto minmax(80px, 100%);
+                grid-template-rows: 1fr auto auto auto auto;
             }
             body > * {
                 grid-column: 1;
@@ -281,9 +327,14 @@
                 grid-template-columns: auto;
             }
 
+            div.section-manage-files,
             div.api-settings {
                 width: auto;
             }
+
+            div.finalize-buttons {
+                padding: 0;
+            }
         }
     </style>
 </html>

src/khoj/interface/web/config.html

@@ -67,130 +67,6 @@
                     </div>
                 {% endif %}
             </div>
-            <div class="card">
-                <div class="card-title-row">
-                    <img class="card-icon" src="/static/assets/icons/markdown.svg" alt="markdown">
-                    <h3 class="card-title">
-                        Markdown
-                        {% if current_model_state.markdown == True%}
-                            <img id="configured-icon-markdown" class="configured-icon" src="/static/assets/icons/confirm-icon.svg" alt="Configured">
-                        {% endif %}
-                    </h3>
-                </div>
-                <div class="card-description-row">
-                    <p class="card-description">Set markdown files to index</p>
-                </div>
-                <div class="card-action-row">
-                    <a class="card-button" href="/config/content_type/markdown">
-                        {% if current_model_state.markdown %}
-                            Update
-                        {% else %}
-                            Setup
-                        {% endif %}
-                        <svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14M12 5l7 7-7 7"></path></svg>
-                    </a>
-                </div>
-                {% if current_model_state.markdown %}
-                    <div id="clear-markdown" class="card-action-row">
-                        <button class="card-button" onclick="clearContentType('markdown')">
-                            Disable
-                        </button>
-                    </div>
-                {% endif %}
-            </div>
-            <div class="card">
-                <div class="card-title-row">
-                    <img class="card-icon" src="/static/assets/icons/org.svg" alt="org">
-                    <h3 class="card-title">
-                        Org
-                        {% if current_model_state.org == True %}
-                            <img id="configured-icon-org" class="configured-icon" src="/static/assets/icons/confirm-icon.svg" alt="Configured">
-                        {% endif %}
-                    </h3>
-                </div>
-                <div class="card-description-row">
-                <p class="card-description">Set org files to index</p>
-                </div>
-                <div class="card-action-row">
-                    <a class="card-button" href="/config/content_type/org">
-                        {% if current_model_state.org %}
-                            Update
-                        {% else %}
-                            Setup
-                        {% endif %}
-                        <svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14M12 5l7 7-7 7"></path></svg>
-                    </a>
-                </div>
-                {% if current_model_state.org %}
-                    <div id="clear-org" class="card-action-row">
-                        <button class="card-button" onclick="clearContentType('org')">
-                            Disable
-                        </button>
-                    </div>
-                {% endif %}
-            </div>
-            <div class="card">
-                <div class="card-title-row">
-                    <img class="card-icon" src="/static/assets/icons/pdf.svg" alt="PDF">
-                    <h3 class="card-title">
-                        PDF
-                        {% if current_model_state.pdf == True %}
-                            <img id="configured-icon-pdf" class="configured-icon" src="/static/assets/icons/confirm-icon.svg" alt="Configured">
-                        {% endif %}
-                    </h3>
-                </div>
-                <div class="card-description-row">
-                <p class="card-description">Set PDF files to index</p>
-                </div>
-                <div class="card-action-row">
-                    <a class="card-button" href="/config/content_type/pdf">
-                        {% if current_model_state.pdf %}
-                            Update
-                        {% else %}
-                            Setup
-                        {% endif %}
-                        <svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14M12 5l7 7-7 7"></path></svg>
-                    </a>
-                </div>
-                {% if current_model_state.pdf %}
-                    <div id="clear-pdf" class="card-action-row">
-                        <button class="card-button" onclick="clearContentType('pdf')">
-                            Disable
-                        </button>
-                    </div>
-                {% endif %}
-            </div>
-            <div class="card">
-                <div class="card-title-row">
-                    <img class="card-icon" src="/static/assets/icons/plaintext.svg" alt="Plaintext">
-                    <h3 class="card-title">
-                        Plaintext
-                        {% if current_model_state.plaintext == True %}
-                            <img id="configured-icon-plaintext" class="configured-icon" src="/static/assets/icons/confirm-icon.svg" alt="Configured">
-                        {% endif %}
-                    </h3>
-                </div>
-                <div class="card-description-row">
-                <p class="card-description">Set Plaintext files to index</p>
-                </div>
-                <div class="card-action-row">
-                    <a class="card-button" href="/config/content_type/plaintext">
-                        {% if current_model_state.plaintext %}
-                            Update
-                        {% else %}
-                            Setup
-                        {% endif %}
-                        <svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14M12 5l7 7-7 7"></path></svg>
-                    </a>
-                </div>
-                {% if current_model_state.plaintext %}
-                    <div id="clear-plaintext" class="card-action-row">
-                        <button class="card-button" onclick="clearContentType('plaintext')">
-                            Disable
-                        </button>
-                    </div>
-                {% endif %}
-            </div>
         </div>
     </div>
     <div class="section">
@@ -246,6 +122,16 @@
             </div>
         </div>
     </div>
+    <div class="section">
+        <h2 class="section-title">Manage Data</h2>
+        <div class="section-manage-files">
+            <div id="delete-all-files" class="delete-all=files">
+                <button id="delete-all-files" type="submit" title="Delete all indexed files">🗑️ Remove All</button>
+            </div>
+            <div class="indexed-files">
+            </div>
+        </div>
+    </div>
     <div class="section general-settings">
         <div id="results-count" title="Number of items to show in search and use for chat response">
             <label for="results-count-slider">Results Count: <span id="results-count-value">5</span></label>
@@ -291,8 +177,8 @@
     };
 
     function clearContentType(content_type) {
-        fetch('/api/delete/config/data/content_type/' + content_type, {
-            method: 'POST',
+        fetch('/api/config/data/content_type/' + content_type, {
+            method: 'DELETE',
             headers: {
                 'Content-Type': 'application/json',
             }
@@ -462,5 +348,84 @@
     // List user's API keys on page load
     listApiKeys();
 
+    function removeFile(path) {
+        fetch('/api/config/data/file?filename=' + path, {
+            method: 'DELETE',
+            headers: {
+                'Content-Type': 'application/json',
+            }
+        })
+        .then(response => response.json())
+        .then(data => {
+            if (data.status == "ok") {
+                getAllFilenames();
+            }
+        })
+    }
+
+    // Get all currently indexed files
+    function getAllFilenames() {
+        fetch('/api/config/data/all')
+            .then(response => response.json())
+            .then(data => {
+                var indexedFiles = document.getElementsByClassName("indexed-files")[0];
+                indexedFiles.innerHTML = "";
+
+                if (data.length == 0) {
+                    document.getElementById("delete-all-files").style.display = "none";
+                    indexedFiles.innerHTML = "<div>Use the <a href='https://download.khoj.dev'>Khoj Desktop client</a> to index files.</div>";
+                } else {
+                    document.getElementById("delete-all-files").style.display = "block";
+                }
+
+                for (var filename of data) {
+                    let fileElement = document.createElement("div");
+                    fileElement.classList.add("file-element");
+
+                    let fileNameElement = document.createElement("div");
+                    fileNameElement.classList.add("content-name");
+                    fileNameElement.innerHTML = filename;
+                    fileElement.appendChild(fileNameElement);
+
+                    let buttonContainer = document.createElement("div");
+                    buttonContainer.classList.add("remove-button-container");
+                    let removeFileButton = document.createElement("button");
+                    removeFileButton.classList.add("remove-file-button");
+                    removeFileButton.innerHTML = "🗑️";
+                    removeFileButton.addEventListener("click", ((filename) => {
+                        return () => {
+                            removeFile(filename);
+                        };
+                    })(filename));
+                    buttonContainer.appendChild(removeFileButton);
+                    fileElement.appendChild(buttonContainer);
+                    indexedFiles.appendChild(fileElement);
+                }
+            })
+            .catch((error) => {
+                console.error('Error:', error);
+            });
+    }
+
+    // Get all currently indexed files on page load
+    getAllFilenames();
+
+    let deleteAllFilesButton = document.getElementById("delete-all-files");
+    deleteAllFilesButton.addEventListener("click", function(event) {
+        event.preventDefault();
+        fetch('/api/config/data/all', {
+            method: 'DELETE',
+            headers: {
+                'Content-Type': 'application/json',
+            }
+        })
+        .then(response => response.json())
+        .then(data => {
+            if (data.status == "ok") {
+                getAllFilenames();
+            }
+        })
+    });
+
 </script>
 {% endblock %}

src/khoj/routers/api.py

@@ -45,7 +45,15 @@ from fastapi.requests import Request
 
 from database import adapters
 from database.adapters import EntryAdapters, ConversationAdapters
-from database.models import LocalMarkdownConfig, LocalOrgConfig, LocalPdfConfig, LocalPlaintextConfig, KhojUser
+from database.models import (
+    LocalMarkdownConfig,
+    LocalOrgConfig,
+    LocalPdfConfig,
+    LocalPlaintextConfig,
+    KhojUser,
+    GithubConfig,
+    NotionConfig,
+)
 
 
 # Initialize Router
@@ -54,14 +62,10 @@ logger = logging.getLogger(__name__)
 
 
 def map_config_to_object(content_type: str):
-    if content_type == "org":
-        return LocalOrgConfig
-    if content_type == "markdown":
-        return LocalMarkdownConfig
-    if content_type == "pdf":
-        return LocalPdfConfig
-    if content_type == "plaintext":
-        return LocalPlaintextConfig
+    if content_type == "github":
+        return GithubConfig
+    if content_type == "notion":
+        return NotionConfig
 
 
 async def map_config_to_db(config: FullConfig, user: KhojUser):
@@ -215,7 +219,7 @@ async def set_content_config_notion_data(
     return {"status": "ok"}
 
 
-@api.post("/delete/config/data/content_type/{content_type}", status_code=200)
+@api.delete("/config/data/content_type/{content_type}", status_code=200)
 @requires(["authenticated"])
 async def remove_content_config_data(
     request: Request,
@@ -243,29 +247,62 @@ async def remove_content_config_data(
     return {"status": "ok"}
 
 
-@api.post("/config/data/content_type/{content_type}", status_code=200)
+@api.delete("/config/data/file", status_code=200)
 @requires(["authenticated"])
-async def set_content_config_data(
+async def remove_file_data(
     request: Request,
-    content_type: str,
-    updated_config: Union[TextContentConfig, None],
+    filename: str,
     client: Optional[str] = None,
 ):
-    _initialize_config()
-
     user = request.user.object
 
-    content_object = map_config_to_object(content_type)
-    await adapters.set_text_content_config(user, content_object, updated_config)
-
     update_telemetry_state(
         request=request,
         telemetry_type="api",
-        api="set_content_config",
+        api="delete_file",
         client=client,
-        metadata={"content_type": content_type},
     )
 
+    await EntryAdapters.adelete_entry_by_file(user, filename)
+
+    return {"status": "ok"}
+
+
+@api.get("/config/data/all", response_model=List[str])
+@requires(["authenticated"])
+async def get_all_filenames(
+    request: Request,
+    client: Optional[str] = None,
+):
+    user = request.user.object
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="get_all_filenames",
+        client=client,
+    )
+
+    return await sync_to_async(list)(EntryAdapters.aget_all_filenames(user))
+
+
+@api.delete("/config/data/all", status_code=200)
+@requires(["authenticated"])
+async def remove_all_config_data(
+    request: Request,
+    client: Optional[str] = None,
+):
+    user = request.user.object
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="delete_all_config",
+        client=client,
+    )
+
+    await EntryAdapters.adelete_all_entries(user)
+
     return {"status": "ok"}