Add a unique_id field for identifiying conversations (#914)

* Add a unique_id field to the conversation object

- This helps us keep track of the unique identity of the conversation without expose the internal id
- Create three staged migrations in order to first add the field, then add unique values to pre-fill, and then set the unique constraint. Without this, it tries to initialize all the existing conversations with the same ID.

* Parse and utilize the unique_id field in the query parameters of the front-end view

- Handle the unique_id field when creating a new conversation from the home page
- Parse the id field with a lightweight parameter called v in the chat page
- Share page should not be affected, as it uses the public slug

* Fix suggested card category

src/interface/web/app/chat/page.tsx

@@ -32,7 +32,8 @@ interface ChatBodyDataProps {
 
 function ChatBodyData(props: ChatBodyDataProps) {
     const searchParams = useSearchParams();
-    const conversationId = searchParams.get("conversationId");
+    const conversationUniqueId = searchParams.get("v");
+    const [conversationId, setConversationId] = useState<string | null>("");
     const [message, setMessage] = useState("");
     const [image, setImage] = useState<string | null>(null);
     const [processingMessage, setProcessingMessage] = useState(false);
@@ -60,6 +61,11 @@ function ChatBodyData(props: ChatBodyDataProps) {
             setProcessingMessage(true);
             setQueryToProcess(storedMessage);
         }
+
+        const conversationId = localStorage.getItem("conversationId");
+        if (conversationId) {
+            setConversationId(conversationId);
+        }
     }, [setQueryToProcess]);
 
     useEffect(() => {
@@ -69,6 +75,30 @@ function ChatBodyData(props: ChatBodyDataProps) {
         }
     }, [message, setQueryToProcess]);
 
+    useEffect(() => {
+        if (!conversationUniqueId) {
+            return;
+        }
+
+        fetch(
+            `/api/chat/metadata?conversation_unique_id=${encodeURIComponent(conversationUniqueId)}`,
+        )
+            .then((response) => {
+                if (!response.ok) {
+                    throw new Error(response.statusText);
+                }
+                return response.json();
+            })
+            .then((data) => {
+                setConversationId(data.conversationId);
+            })
+            .catch((err) => {
+                console.error(err);
+                setConversationId(null);
+                return;
+            });
+    });
+
     useEffect(() => {
         if (conversationId) {
             onConversationIdChange?.(conversationId);
@@ -87,11 +117,15 @@ function ChatBodyData(props: ChatBodyDataProps) {
         }
     }, [props.streamedMessages]);
 
-    if (!conversationId) {
+    if (!conversationUniqueId || conversationId === null) {
         window.location.href = "/";
         return;
     }
 
+    if (!conversationId) {
+        return <Loading />;
+    }
+
     return (
         <>
             <div className={false ? styles.chatBody : styles.chatBodyFull}>

src/interface/web/app/common/chatFunctions.ts

@@ -186,6 +186,11 @@ export function modifyFileFilterForConversation(
         });
 }
 
+interface NewConversationMetadata {
+    conversationId: string;
+    conversationUniqueId: string;
+}
+
 export async function createNewConversation(slug: string) {
     try {
         const response = await fetch(`/api/chat/sessions?client=web&agent_slug=${slug}`, {
@@ -194,9 +199,11 @@ export async function createNewConversation(slug: string) {
         if (!response.ok)
             throw new Error(`Failed to fetch chat sessions with status: ${response.status}`);
         const data = await response.json();
-        const conversationID = data.conversation_id;
+        const uniqueId = data.unique_id;
-        if (!conversationID) throw new Error("Conversation ID not found in response");
+        const conversationId = data.conversation_id;
-        return conversationID;
+        if (!uniqueId) throw new Error("Unique ID not found in response");
+        if (!conversationId) throw new Error("Conversation ID not found in response");
+        return { conversationId, conversationUniqueId: uniqueId } as NewConversationMetadata;
     } catch (error) {
         console.error("Error creating new conversation:", error);
         throw error;

src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx

@@ -67,7 +67,9 @@ interface ChatHistory {
     compressed: boolean;
     created: string;
     updated: string;
+    unique_id: string;
     showSidePanel: (isEnabled: boolean) => void;
+    selectedConversationId: string | null;
 }
 
 import {
@@ -398,6 +400,7 @@ interface SessionsAndFilesProps {
     conversationId: string | null;
     uploadedFiles: string[];
     isMobileWidth: boolean;
+    selectedConversationId: string | null;
 }
 
 function SessionsAndFiles(props: SessionsAndFilesProps) {
@@ -435,6 +438,10 @@ function SessionsAndFiles(props: SessionsAndFilesProps) {
                                                         agent_avatar={chatHistory.agent_avatar}
                                                         agent_name={chatHistory.agent_name}
                                                         showSidePanel={props.setEnabled}
+                                                        unique_id={chatHistory.unique_id}
+                                                        selectedConversationId={
+                                                            props.selectedConversationId
+                                                        }
                                                     />
                                                 ),
                                             )}
@@ -446,6 +453,7 @@ function SessionsAndFiles(props: SessionsAndFilesProps) {
                     <ChatSessionsModal
                         data={props.organizedData}
                         showSidePanel={props.setEnabled}
+                        selectedConversationId={props.selectedConversationId}
                     />
                 )}
             </div>
@@ -640,20 +648,18 @@ function ChatSessionActionMenu(props: ChatSessionActionMenuProps) {
 function ChatSession(props: ChatHistory) {
     const [isHovered, setIsHovered] = useState(false);
     const [title, setTitle] = useState(props.slug || "New Conversation 🌱");
-    var currConversationId = parseInt(
+    var currConversationId =
-        new URLSearchParams(window.location.search).get("conversationId") || "-1",
+        props.conversation_id &&
-    );
+        props.selectedConversationId &&
+        parseInt(props.conversation_id) === parseInt(props.selectedConversationId);
     return (
         <div
             onMouseEnter={() => setIsHovered(true)}
             onMouseLeave={() => setIsHovered(false)}
             key={props.conversation_id}
-            className={`${styles.session} ${props.compressed ? styles.compressed : "!max-w-full"} ${isHovered ? `${styles.sessionHover}` : ""} ${currConversationId === parseInt(props.conversation_id) && currConversationId != -1 ? "dark:bg-neutral-800 bg-white" : ""}`}
+            className={`${styles.session} ${props.compressed ? styles.compressed : "!max-w-full"} ${isHovered ? `${styles.sessionHover}` : ""} ${currConversationId ? "dark:bg-neutral-800 bg-white" : ""}`}
         >
-            <Link
+            <Link href={`/chat?v=${props.unique_id}`} onClick={() => props.showSidePanel(false)}>
-                href={`/chat?conversationId=${props.conversation_id}`}
-                onClick={() => props.showSidePanel(false)}
-            >
                 <p className={styles.session}>{title}</p>
             </Link>
             <ChatSessionActionMenu conversationId={props.conversation_id} setTitle={setTitle} />
@@ -664,9 +670,14 @@ function ChatSession(props: ChatHistory) {
 interface ChatSessionsModalProps {
     data: GroupedChatHistory | null;
     showSidePanel: (isEnabled: boolean) => void;
+    selectedConversationId: string | null;
 }
 
-function ChatSessionsModal({ data, showSidePanel }: ChatSessionsModalProps) {
+function ChatSessionsModal({
+    data,
+    showSidePanel,
+    selectedConversationId,
+}: ChatSessionsModalProps) {
     return (
         <Dialog>
             <DialogTrigger className="flex text-left text-medium text-gray-500 hover:text-gray-300 cursor-pointer my-4 text-sm p-[0.5rem]">
@@ -698,6 +709,8 @@ function ChatSessionsModal({ data, showSidePanel }: ChatSessionsModalProps) {
                                                 agent_avatar={chatHistory.agent_avatar}
                                                 agent_name={chatHistory.agent_name}
                                                 showSidePanel={showSidePanel}
+                                                unique_id={chatHistory.unique_id}
+                                                selectedConversationId={selectedConversationId}
                                             />
                                         ))}
                                     </div>
@@ -819,6 +832,7 @@ export default function SidePanel(props: SidePanelProps) {
                                         userProfile={authenticatedData}
                                         conversationId={props.conversationId}
                                         isMobileWidth={props.isMobileWidth}
+                                        selectedConversationId={props.conversationId}
                                     />
                                 </div>
                             ) : (
@@ -887,6 +901,7 @@ export default function SidePanel(props: SidePanelProps) {
                         userProfile={authenticatedData}
                         conversationId={props.conversationId}
                         isMobileWidth={props.isMobileWidth}
+                        selectedConversationId={props.conversationId}
                     />
                 </div>
             )}

src/interface/web/app/components/suggestions/suggestionsData.ts

@@ -408,7 +408,7 @@ export const suggestionsData: Suggestion[] = [
         link: "",
     },
     {
-        type: SuggestionType.Code,
+        type: SuggestionType.Interviewing,
         color: suggestionToColorMap[SuggestionType.Interviewing] || DEFAULT_COLOR,
         description: "Provide tips for writing an effective resume.",
         link: "",

src/interface/web/app/page.tsx

@@ -138,10 +138,13 @@ function ChatBodyData(props: ChatBodyDataProps) {
             if (message && !processingMessage) {
                 setProcessingMessage(true);
                 try {
-                    const newConversationId = await createNewConversation(selectedAgent || "khoj");
+                    const newConversationMetadata = await createNewConversation(
-                    onConversationIdChange?.(newConversationId);
+                        selectedAgent || "khoj",
-                    window.location.href = `/chat?conversationId=${newConversationId}`;
+                    );
+                    onConversationIdChange?.(newConversationMetadata.conversationId);
+                    window.location.href = `/chat?v=${newConversationMetadata.conversationUniqueId}`;
                     localStorage.setItem("message", message);
+                    localStorage.setItem("conversationId", newConversationMetadata.conversationId);
                     if (image) {
                         localStorage.setItem("image", image);
                     }

src/khoj/database/adapters/__init__.py

@@ -679,6 +679,10 @@ class ConversationAdapters:
 
         return conversation
 
+    @staticmethod
+    def get_conversation_by_unique_id(user: KhojUser, unique_id: str):
+        return Conversation.objects.filter(unique_id=unique_id, user=user).first()
+
     @staticmethod
     def get_conversation_sessions(user: KhojUser, client_application: ClientApplication = None):
         return (

src/khoj/database/migrations/0063_conversation_add_unique_id_field.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.8 on 2024-09-16 04:12
+
+import uuid
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0062_merge_20240913_0222"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="conversation",
+            name="unique_id",
+            field=models.UUIDField(default=None, editable=False, null=True),
+        ),
+    ]

src/khoj/database/migrations/0064_populate_unique_id.py

@@ -0,0 +1,20 @@
+import uuid
+
+from django.db import migrations
+
+
+def populate_unique_id(apps, schema_editor):
+    Conversation = apps.get_model("database", "Conversation")
+    for conversation in Conversation.objects.all():
+        conversation.unique_id = uuid.uuid4()
+        conversation.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0063_conversation_add_unique_id_field"),
+    ]
+
+    operations = [
+        migrations.RunPython(populate_unique_id),
+    ]

src/khoj/database/migrations/0065_add_unique_constraint_to_unique_id.py

@@ -0,0 +1,17 @@
+import uuid
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0064_populate_unique_id"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="conversation",
+            name="unique_id",
+            field=models.UUIDField(default=uuid.uuid4, editable=False, unique=True),
+        ),
+    ]

src/khoj/database/models/__init__.py

@@ -350,6 +350,7 @@ class Conversation(BaseModel):
     title = models.CharField(max_length=200, default=None, null=True, blank=True)
     agent = models.ForeignKey(Agent, on_delete=models.SET_NULL, default=None, null=True, blank=True)
     file_filters = models.JSONField(default=list)
+    unique_id = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)
 
 
 class PublicConversation(BaseModel):

src/khoj/routers/api_chat.py

@@ -224,6 +224,7 @@ def chat_history(
             "conversation_id": conversation.id,
             "slug": conversation.title if conversation.title else conversation.slug,
             "agent": agent_metadata,
+            "unique_id": conversation.unique_id,
         }
     )
 
@@ -245,6 +246,33 @@ def chat_history(
     return {"status": "ok", "response": meta_log}
 
 
+@api_chat.get("/metadata")
+def get_chat_metadata(
+    request: Request,
+    common: CommonQueryParams,
+    conversation_unique_id: str,
+):
+    user = request.user.object
+
+    # Load Conversation Metadata
+    conversation = ConversationAdapters.get_conversation_by_unique_id(user, conversation_unique_id)
+
+    if conversation is None:
+        return Response(
+            content=json.dumps({"status": "error", "message": f"Conversation: {conversation_unique_id} not found"}),
+            status_code=404,
+        )
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="chat_metadata",
+        **common.__dict__,
+    )
+
+    return {"status": "ok", "conversationId": conversation.id}
+
+
 @api_chat.get("/share/history")
 def get_shared_chat(
     request: Request,
@@ -418,7 +446,7 @@ def chat_sessions(
         conversations = conversations[:8]
 
     sessions = conversations.values_list(
-        "id", "slug", "title", "agent__slug", "agent__name", "agent__avatar", "created_at", "updated_at"
+        "id", "slug", "title", "agent__slug", "agent__name", "agent__avatar", "created_at", "updated_at", "unique_id"
     )
 
     session_values = [
@@ -429,6 +457,7 @@ def chat_sessions(
             "agent_avatar": session[5],
             "created": session[6].strftime("%Y-%m-%d %H:%M:%S"),
             "updated": session[7].strftime("%Y-%m-%d %H:%M:%S"),
+            "unique_id": str(session[8]),
         }
         for session in sessions
     ]
@@ -455,7 +484,7 @@ async def create_chat_session(
     # Create new Conversation Session
     conversation = await ConversationAdapters.acreate_conversation_session(user, request.user.client_app, agent_slug)
 
-    response = {"conversation_id": conversation.id}
+    response = {"conversation_id": conversation.id, "unique_id": str(conversation.unique_id)}
 
     conversation_metadata = {
         "agent": agent_slug,