klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-05 05:39:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #552 from khoj-ai/features/internet-enabled-search

Browse Source 
Support internet-enabled, online searching using Serper.dev
This commit is contained in:
sabaimran
2023-11-23 12:34:05 -08:00
committed by GitHub
parent 5fac39afed e3b32e412c
commit c42ec32a95
42 changed files with 998 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

199
src/interface/desktop/chat.html
View File

@@ -60,6 +60,52 @@
return referenceButton;
}
function generateOnlineReference(reference, index) {
// Generate HTML for Chat Reference
let title = reference.title;
let link = reference.link;
let snippet = reference.snippet;
let question = reference.question;
if (question) {
question = `<b>Question:</b> ${question}<br><br>`;
} else {
question = "";
}
let linkElement = document.createElement('a');
linkElement.setAttribute('href', link);
linkElement.setAttribute('target', '_blank');
linkElement.setAttribute('rel', 'noopener noreferrer');
linkElement.classList.add("inline-chat-link");
linkElement.classList.add("reference-link");
linkElement.setAttribute('title', title);
linkElement.innerHTML = title;
let referenceButton = document.createElement('button');
referenceButton.innerHTML = linkElement.outerHTML;
referenceButton.id = `ref-${index}`;
referenceButton.classList.add("reference-button");
referenceButton.classList.add("collapsed");
referenceButton.tabIndex = 0;
// Add event listener to toggle full reference on click
referenceButton.addEventListener('click', function() {
console.log(`Toggling ref-${index}`)
if (this.classList.contains("collapsed")) {
this.classList.remove("collapsed");
this.classList.add("expanded");
this.innerHTML = linkElement.outerHTML + `<br><br>${question + snippet}`;
} else {
this.classList.add("collapsed");
this.classList.remove("expanded");
this.innerHTML = linkElement.outerHTML;
}
});
return referenceButton;
}
function renderMessage(message, by, dt=null, annotations=null) {
let message_time = formatDate(dt ?? new Date());
let by_name = by == "khoj" ? "🏮 Khoj" : "🤔 You";
@@ -90,8 +136,48 @@
chatBody.scrollTop = chatBody.scrollHeight;
}
function renderMessageWithReference(message, by, context=null, dt=null) {
if (context == null || context.length == 0) {
function processOnlineReferences(referenceSection, onlineContext) {
let numOnlineReferences = 0;
for (let subquery in onlineContext) {
let onlineReference = onlineContext[subquery];
if (onlineReference.organic && onlineReference.organic.length > 0) {
numOnlineReferences += onlineReference.organic.length;
for (let index in onlineReference.organic) {
let reference = onlineReference.organic[index];
let polishedReference = generateOnlineReference(reference, index);
referenceSection.appendChild(polishedReference);
}
}
if (onlineReference.knowledgeGraph && onlineReference.knowledgeGraph.length > 0) {
numOnlineReferences += onlineReference.knowledgeGraph.length;
for (let index in onlineReference.knowledgeGraph) {
let reference = onlineReference.knowledgeGraph[index];
let polishedReference = generateOnlineReference(reference, index);
referenceSection.appendChild(polishedReference);
}
}
if (onlineReference.peopleAlsoAsk && onlineReference.peopleAlsoAsk.length > 0) {
numOnlineReferences += onlineReference.peopleAlsoAsk.length;
for (let index in onlineReference.peopleAlsoAsk) {
let reference = onlineReference.peopleAlsoAsk[index];
let polishedReference = generateOnlineReference(reference, index);
referenceSection.appendChild(polishedReference);
}
}
}
return numOnlineReferences;
}
function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null) {
if (context == null && onlineContext == null) {
renderMessage(message, by, dt);
return;
}
if ((context && context.length == 0) && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) {
renderMessage(message, by, dt);
return;
}
@@ -100,8 +186,11 @@
let referenceExpandButton = document.createElement('button');
referenceExpandButton.classList.add("reference-expand-button");
let expandButtonText = context.length == 1 ? "1 reference" : `${context.length} references`;
referenceExpandButton.innerHTML = expandButtonText;
let numReferences = 0;
if (context) {
numReferences += context.length;
}
references.appendChild(referenceExpandButton);
@@ -127,6 +216,14 @@
referenceSection.appendChild(polishedReference);
}
}
if (onlineContext) {
numReferences += processOnlineReferences(referenceSection, onlineContext);
}
let expandButtonText = numReferences == 1 ? "1 reference" : `${numReferences} references`;
referenceExpandButton.innerHTML = expandButtonText;
references.appendChild(referenceSection);
renderMessage(message, by, dt, references);
@@ -140,6 +237,8 @@
newHTML = newHTML.replace(/__([\s\S]*?)__/g, '<u>$1</u>');
// Remove any text between <s>[INST] and </s> tags. These are spurious instructions for the AI chat model.
newHTML = newHTML.replace(/<s>\[INST\].+(<\/s>)?/g, '');
// For any text that has single backticks, replace them with <code> tags
newHTML = newHTML.replace(/`([^`]+)`/g, '<code class="chat-response">$1</code>');
return newHTML;
}
@@ -221,15 +320,28 @@
let referenceExpandButton = document.createElement('button');
referenceExpandButton.classList.add("reference-expand-button");
let expandButtonText = rawReferenceAsJson.length == 1 ? "1 reference" : `${rawReferenceAsJson.length} references`;
referenceExpandButton.innerHTML = expandButtonText;
references.appendChild(referenceExpandButton);
let referenceSection = document.createElement('div');
referenceSection.classList.add("reference-section");
referenceSection.classList.add("collapsed");
let numReferences = 0;
// If rawReferenceAsJson is a list, then count the length
if (Array.isArray(rawReferenceAsJson)) {
numReferences = rawReferenceAsJson.length;
rawReferenceAsJson.forEach((reference, index) => {
let polishedReference = generateReference(reference, index);
referenceSection.appendChild(polishedReference);
});
} else {
numReferences += processOnlineReferences(referenceSection, rawReferenceAsJson);
}
references.appendChild(referenceExpandButton);
referenceExpandButton.addEventListener('click', function() {
if (referenceSection.classList.contains("collapsed")) {
referenceSection.classList.remove("collapsed");
@@ -240,10 +352,8 @@
}
});
rawReferenceAsJson.forEach((reference, index) => {
let polishedReference = generateReference(reference, index);
referenceSection.appendChild(polishedReference);
});
let expandButtonText = numReferences == 1 ? "1 reference" : `${numReferences} references`;
referenceExpandButton.innerHTML = expandButtonText;
references.appendChild(referenceSection);
readStream();
} else {
@@ -276,6 +386,9 @@
let chatInput = document.getElementById("chat-input");
chatInput.value = chatInput.value.trimStart();
let questionStarterSuggestions = document.getElementById("question-starters");
questionStarterSuggestions.style.display = "none";
if (chatInput.value.startsWith("/") && chatInput.value.split(" ").length === 1) {
let chatTooltip = document.getElementById("chat-tooltip");
chatTooltip.style.display = "block";
@@ -324,7 +437,7 @@
const khojToken = await window.tokenAPI.getToken();
const headers = { 'Authorization': `Bearer ${khojToken}` };
fetch(`${hostURL}/api/chat/history?client=web`, { headers })
fetch(`${hostURL}/api/chat/history?client=desktop`, { headers })
.then(response => response.json())
.then(data => {
if (data.detail) {
@@ -351,13 +464,38 @@
.then(response => {
// Render conversation history, if any
response.forEach(chat_log => {
renderMessageWithReference(chat_log.message, chat_log.by, chat_log.context, new Date(chat_log.created));
renderMessageWithReference(chat_log.message, chat_log.by, chat_log.context, new Date(chat_log.created), chat_log.onlineContext);
});
})
.catch(err => {
return;
});
fetch(`${hostURL}/api/chat/starters?client=desktop`, { headers })
.then(response => response.json())
.then(data => {
// Render chat options, if any
if (data) {
let questionStarterSuggestions = document.getElementById("question-starters");
for (let index in data) {
let questionStarter = data[index];
let questionStarterButton = document.createElement('button');
questionStarterButton.innerHTML = questionStarter;
questionStarterButton.classList.add("question-starter");
questionStarterButton.addEventListener('click', function() {
questionStarterSuggestions.style.display = "none";
document.getElementById("chat-input").value = questionStarter;
chat();
});
questionStarterSuggestions.appendChild(questionStarterButton);
}
questionStarterSuggestions.style.display = "grid";
}
})
.catch(err => {
return;
});
fetch(`${hostURL}/api/chat/options`, { headers })
.then(response => response.json())
.then(data => {
@@ -397,6 +535,9 @@
<!-- Chat Body -->
<div id="chat-body"></div>
<!-- Chat Suggestions -->
<div id="question-starters" style="display: none;"></div>
<!-- Chat Footer -->
<div id="chat-footer">
<div id="chat-tooltip" style="display: none;"></div>
@@ -574,6 +715,38 @@
margin: 10px;
}
div#question-starters {
grid-template-columns: repeat(auto-fit, minmax(100px, 1fr));
grid-column-gap: 8px;
}
button.question-starter {
background: var(--background-color);
color: var(--main-text-color);
border: 1px solid var(--main-text-color);
border-radius: 5px;
padding: 5px;
font-size: 14px;
font-weight: 300;
line-height: 1.5em;
cursor: pointer;
transition: background 0.2s ease-in-out;
text-align: left;
max-height: 75px;
transition: max-height 0.3s ease-in-out;
overflow: hidden;
}
code.chat-response {
background: var(--primary-hover);
color: var(--primary-inverse);
border-radius: 5px;
padding: 5px;
font-size: 14px;
font-weight: 300;
line-height: 1.5em;
}
button.reference-button {
background: var(--background-color);
color: var(--main-text-color);