mirror of
https://github.com/khoaliber/khoj.git
synced 2026-03-07 21:29:13 +00:00
Rename notes search type to org search, now that markdown notes supported
This commit is contained in:
@@ -114,9 +114,9 @@
|
|||||||
(cond
|
(cond
|
||||||
((equal buffer-name "Music.org") "music")
|
((equal buffer-name "Music.org") "music")
|
||||||
((equal file-extension "bean") "ledger")
|
((equal file-extension "bean") "ledger")
|
||||||
((equal file-extension "org") "notes")
|
((equal file-extension "org") "org")
|
||||||
((or (equal file-extension "markdown") (equal file-extension "md")) "markdown")
|
((or (equal file-extension "markdown") (equal file-extension "md")) "markdown")
|
||||||
(t "notes"))))
|
(t "org"))))
|
||||||
|
|
||||||
(defun khoj--construct-api-query (query search-type)
|
(defun khoj--construct-api-query (query search-type)
|
||||||
(let ((encoded-query (url-hexify-string query)))
|
(let ((encoded-query (url-hexify-string query)))
|
||||||
@@ -127,7 +127,7 @@
|
|||||||
"Search your content naturally using the Khoj API"
|
"Search your content naturally using the Khoj API"
|
||||||
(interactive "sQuery: ")
|
(interactive "sQuery: ")
|
||||||
(let* ((default-type (khoj--buffer-name-to-search-type (buffer-name)))
|
(let* ((default-type (khoj--buffer-name-to-search-type (buffer-name)))
|
||||||
(search-type (completing-read "Type: " '("notes" "markdown" "ledger" "music" "image") nil t default-type))
|
(search-type (completing-read "Type: " '("org" "markdown" "ledger" "music" "image") nil t default-type))
|
||||||
(url (khoj--construct-api-query query search-type))
|
(url (khoj--construct-api-query query search-type))
|
||||||
(buff (get-buffer-create (format "*Khoj (q:%s t:%s)*" query search-type))))
|
(buff (get-buffer-create (format "*Khoj (q:%s t:%s)*" query search-type))))
|
||||||
;; get json response from api
|
;; get json response from api
|
||||||
@@ -141,12 +141,12 @@
|
|||||||
(json-response (json-parse-buffer :object-type 'alist)))
|
(json-response (json-parse-buffer :object-type 'alist)))
|
||||||
(erase-buffer)
|
(erase-buffer)
|
||||||
(insert
|
(insert
|
||||||
(cond ((or (equal search-type "notes") (equal search-type "music")) (khoj--extract-entries-as-org json-response query))
|
(cond ((or (equal search-type "org") (equal search-type "music")) (khoj--extract-entries-as-org json-response query))
|
||||||
((equal search-type "markdown") (khoj--extract-entries-as-markdown json-response query))
|
((equal search-type "markdown") (khoj--extract-entries-as-markdown json-response query))
|
||||||
((equal search-type "ledger") (khoj--extract-entries-as-ledger json-response query))
|
((equal search-type "ledger") (khoj--extract-entries-as-ledger json-response query))
|
||||||
((equal search-type "image") (khoj--extract-entries-as-images json-response query))
|
((equal search-type "image") (khoj--extract-entries-as-images json-response query))
|
||||||
(t (format "%s" json-response))))
|
(t (format "%s" json-response))))
|
||||||
(cond ((equal search-type "notes") (org-mode))
|
(cond ((equal search-type "org") (org-mode))
|
||||||
((equal search-type "markdown") (markdown-mode))
|
((equal search-type "markdown") (markdown-mode))
|
||||||
((equal search-type "ledger") (beancount-mode))
|
((equal search-type "ledger") (beancount-mode))
|
||||||
((equal search-type "music") (progn (org-mode)
|
((equal search-type "music") (progn (org-mode)
|
||||||
|
|||||||
@@ -58,11 +58,11 @@
|
|||||||
<input id="query" type="text" placeholder="Search" onkeydown=search_on_enter(event) autofocus>
|
<input id="query" type="text" placeholder="Search" onkeydown=search_on_enter(event) autofocus>
|
||||||
|
|
||||||
<!--Add Dropdown to Select Query Type.
|
<!--Add Dropdown to Select Query Type.
|
||||||
Query Types can be: notes, ledger, images, music.
|
Query Types can be: org, ledger, image, music, markdown.
|
||||||
Set Default type to images-->
|
Set Default type to image-->
|
||||||
<select id="type">
|
<select id="type">
|
||||||
<option value="image">Image</option>
|
<option value="image">Image</option>
|
||||||
<option value="notes">Notes</option>
|
<option value="org">Org</option>
|
||||||
<option value="markdown">Markdown</option>
|
<option value="markdown">Markdown</option>
|
||||||
<option value="ledger">Ledger</option>
|
<option value="ledger">Ledger</option>
|
||||||
<option value="music">Music</option>
|
<option value="music">Music</option>
|
||||||
|
|||||||
14
src/main.py
14
src/main.py
@@ -67,9 +67,9 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None):
|
|||||||
user_query = q
|
user_query = q
|
||||||
results_count = n
|
results_count = n
|
||||||
|
|
||||||
if (t == SearchType.Notes or t == None) and model.notes_search:
|
if (t == SearchType.Org or t == None) and model.orgmode_search:
|
||||||
# query notes
|
# query org-mode notes
|
||||||
hits, entries = text_search.query(user_query, model.notes_search, device=device, filters=[explicit_filter, date_filter])
|
hits, entries = text_search.query(user_query, model.orgmode_search, device=device, filters=[explicit_filter, date_filter])
|
||||||
|
|
||||||
# collate and return results
|
# collate and return results
|
||||||
return text_search.collate_results(hits, entries, results_count)
|
return text_search.collate_results(hits, entries, results_count)
|
||||||
@@ -81,7 +81,7 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None):
|
|||||||
# collate and return results
|
# collate and return results
|
||||||
return text_search.collate_results(hits, entries, results_count)
|
return text_search.collate_results(hits, entries, results_count)
|
||||||
|
|
||||||
if (t == SearchType.Markdown or t == None) and model.notes_search:
|
if (t == SearchType.Markdown or t == None) and model.orgmode_search:
|
||||||
# query markdown files
|
# query markdown files
|
||||||
hits, entries = text_search.query(user_query, model.markdown_search, device=device, filters=[explicit_filter, date_filter])
|
hits, entries = text_search.query(user_query, model.markdown_search, device=device, filters=[explicit_filter, date_filter])
|
||||||
|
|
||||||
@@ -154,7 +154,7 @@ def chat(q: str):
|
|||||||
|
|
||||||
if get_from_dict(metadata, "intent", "memory-type") == "notes":
|
if get_from_dict(metadata, "intent", "memory-type") == "notes":
|
||||||
query = get_from_dict(metadata, "intent", "query")
|
query = get_from_dict(metadata, "intent", "query")
|
||||||
result_list = search(query, n=1, t=SearchType.Notes)
|
result_list = search(query, n=1, t=SearchType.Org)
|
||||||
collated_result = "\n".join([item["entry"] for item in result_list])
|
collated_result = "\n".join([item["entry"] for item in result_list])
|
||||||
if verbose > 1:
|
if verbose > 1:
|
||||||
print(f'Semantically Similar Notes:\n{collated_result}')
|
print(f'Semantically Similar Notes:\n{collated_result}')
|
||||||
@@ -171,9 +171,9 @@ def chat(q: str):
|
|||||||
|
|
||||||
def initialize_search(config: FullConfig, regenerate: bool, t: SearchType = None, device=torch.device("cpu")):
|
def initialize_search(config: FullConfig, regenerate: bool, t: SearchType = None, device=torch.device("cpu")):
|
||||||
# Initialize Org Notes Search
|
# Initialize Org Notes Search
|
||||||
if (t == SearchType.Notes or t == None) and config.content_type.org:
|
if (t == SearchType.Org or t == None) and config.content_type.org:
|
||||||
# Extract Entries, Generate Notes Embeddings
|
# Extract Entries, Generate Notes Embeddings
|
||||||
model.notes_search = text_search.setup(org_to_jsonl, config.content_type.org, search_config=config.search_type.asymmetric, regenerate=regenerate, device=device, verbose=verbose)
|
model.orgmode_search = text_search.setup(org_to_jsonl, config.content_type.org, search_config=config.search_type.asymmetric, regenerate=regenerate, device=device, verbose=verbose)
|
||||||
|
|
||||||
# Initialize Org Music Search
|
# Initialize Org Music Search
|
||||||
if (t == SearchType.Music or t == None) and config.content_type.music:
|
if (t == SearchType.Music or t == None) and config.content_type.music:
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from src.utils.rawconfig import ConversationProcessorConfig
|
|||||||
|
|
||||||
|
|
||||||
class SearchType(str, Enum):
|
class SearchType(str, Enum):
|
||||||
Notes = "notes"
|
Org = "org"
|
||||||
Ledger = "ledger"
|
Ledger = "ledger"
|
||||||
Music = "music"
|
Music = "music"
|
||||||
Markdown = "markdown"
|
Markdown = "markdown"
|
||||||
@@ -37,7 +37,7 @@ class ImageSearchModel():
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class SearchModels():
|
class SearchModels():
|
||||||
notes_search: TextSearchModel = None
|
orgmode_search: TextSearchModel = None
|
||||||
ledger_search: TextSearchModel = None
|
ledger_search: TextSearchModel = None
|
||||||
music_search: TextSearchModel = None
|
music_search: TextSearchModel = None
|
||||||
markdown_search: TextSearchModel = None
|
markdown_search: TextSearchModel = None
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ def test_search_with_valid_content_type(content_config: ContentConfig, search_co
|
|||||||
config.search_type = search_config
|
config.search_type = search_config
|
||||||
|
|
||||||
# config.content_type.image = search_config.image
|
# config.content_type.image = search_config.image
|
||||||
for content_type in ["notes", "ledger", "music", "image"]:
|
for content_type in ["org", "markdown", "ledger", "music", "image"]:
|
||||||
# Act
|
# Act
|
||||||
response = client.get(f"/search?q=random&t={content_type}")
|
response = client.get(f"/search?q=random&t={content_type}")
|
||||||
# Assert
|
# Assert
|
||||||
@@ -59,7 +59,7 @@ def test_reload_with_valid_content_type(content_config: ContentConfig, search_co
|
|||||||
config.content_type = content_config
|
config.content_type = content_config
|
||||||
config.search_type = search_config
|
config.search_type = search_config
|
||||||
|
|
||||||
for content_type in ["notes", "ledger", "music", "image"]:
|
for content_type in ["org", "markdown", "ledger", "music", "image"]:
|
||||||
# Act
|
# Act
|
||||||
response = client.get(f"/reload?t={content_type}")
|
response = client.get(f"/reload?t={content_type}")
|
||||||
# Assert
|
# Assert
|
||||||
@@ -81,7 +81,7 @@ def test_regenerate_with_valid_content_type(content_config: ContentConfig, searc
|
|||||||
config.content_type = content_config
|
config.content_type = content_config
|
||||||
config.search_type = search_config
|
config.search_type = search_config
|
||||||
|
|
||||||
for content_type in ["notes", "ledger", "music", "image"]:
|
for content_type in ["org", "markdown", "ledger", "music", "image"]:
|
||||||
# Act
|
# Act
|
||||||
response = client.get(f"/regenerate?t={content_type}")
|
response = client.get(f"/regenerate?t={content_type}")
|
||||||
# Assert
|
# Assert
|
||||||
@@ -115,11 +115,11 @@ def test_image_search(content_config: ContentConfig, search_config: SearchConfig
|
|||||||
# ----------------------------------------------------------------------------------------------------
|
# ----------------------------------------------------------------------------------------------------
|
||||||
def test_notes_search(content_config: ContentConfig, search_config: SearchConfig):
|
def test_notes_search(content_config: ContentConfig, search_config: SearchConfig):
|
||||||
# Arrange
|
# Arrange
|
||||||
model.notes_search = text_search.setup(org_to_jsonl, content_config.org, search_config.asymmetric, regenerate=False)
|
model.orgmode_search = text_search.setup(org_to_jsonl, content_config.org, search_config.asymmetric, regenerate=False)
|
||||||
user_query = "How to git install application?"
|
user_query = "How to git install application?"
|
||||||
|
|
||||||
# Act
|
# Act
|
||||||
response = client.get(f"/search?q={user_query}&n=1&t=notes")
|
response = client.get(f"/search?q={user_query}&n=1&t=org")
|
||||||
|
|
||||||
# Assert
|
# Assert
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
@@ -131,11 +131,11 @@ def test_notes_search(content_config: ContentConfig, search_config: SearchConfig
|
|||||||
# ----------------------------------------------------------------------------------------------------
|
# ----------------------------------------------------------------------------------------------------
|
||||||
def test_notes_search_with_include_filter(content_config: ContentConfig, search_config: SearchConfig):
|
def test_notes_search_with_include_filter(content_config: ContentConfig, search_config: SearchConfig):
|
||||||
# Arrange
|
# Arrange
|
||||||
model.notes_search = text_search.setup(org_to_jsonl, content_config.org, search_config.asymmetric, regenerate=False)
|
model.orgmode_search = text_search.setup(org_to_jsonl, content_config.org, search_config.asymmetric, regenerate=False)
|
||||||
user_query = "How to git install application? +Emacs"
|
user_query = "How to git install application? +Emacs"
|
||||||
|
|
||||||
# Act
|
# Act
|
||||||
response = client.get(f"/search?q={user_query}&n=1&t=notes")
|
response = client.get(f"/search?q={user_query}&n=1&t=org")
|
||||||
|
|
||||||
# Assert
|
# Assert
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
@@ -147,11 +147,11 @@ def test_notes_search_with_include_filter(content_config: ContentConfig, search_
|
|||||||
# ----------------------------------------------------------------------------------------------------
|
# ----------------------------------------------------------------------------------------------------
|
||||||
def test_notes_search_with_exclude_filter(content_config: ContentConfig, search_config: SearchConfig):
|
def test_notes_search_with_exclude_filter(content_config: ContentConfig, search_config: SearchConfig):
|
||||||
# Arrange
|
# Arrange
|
||||||
model.notes_search = text_search.setup(org_to_jsonl, content_config.org, search_config.asymmetric, regenerate=False)
|
model.orgmode_search = text_search.setup(org_to_jsonl, content_config.org, search_config.asymmetric, regenerate=False)
|
||||||
user_query = "How to git install application? -clone"
|
user_query = "How to git install application? -clone"
|
||||||
|
|
||||||
# Act
|
# Act
|
||||||
response = client.get(f"/search?q={user_query}&n=1&t=notes")
|
response = client.get(f"/search?q={user_query}&n=1&t=org")
|
||||||
|
|
||||||
# Assert
|
# Assert
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
|
|||||||
Reference in New Issue
Block a user