Deep link to org-mode entries. Deep link by line number in uri

Use url fragment schema for deep link URIs, borrowing from URL/PDF
schemas. E.g file:///path/to/file.txt#line=<line_no>&#page=<page_no>

Compute line number during (recursive) org-mode entry chunking.

Thoroughly test line number in URI maps to line number of chunk in
actual org mode file.

This deeplink URI with line number is passed to llm as context to
better combine with line range based view file tool.

Grep tool already passed matching line number. This change passes
line number in URIs of org entries matched by the semantic search tool

tests/data/org/main_readme.org

@@ -3,7 +3,7 @@
 
   All data is processed locally. User can interface with khoj app via [[./interface/emacs/khoj.el][Emacs]], API or Commandline
 
-** Dependencies
+** Dependencies [[id:123-421-121-12]]  :TAG1:@TAG1_1:
    - Python3
    - [[https://docs.conda.io/en/latest/miniconda.html#latest-miniconda-installer-links][Miniconda]]
 
@@ -22,7 +22,7 @@
    #+end_src
 
 ** Use
-*** *Khoj via Emacs*
+*** *Khoj via Emacs*  [[https://khoj.dev][link to khoj website]]  :@EMACS:CLIENT_1:KHOJ:
      - [[https://github.com/khoj-ai/khoj/tree/master/interface/emacs#installation][Install]] [[./interface/emacs/khoj.el][khoj.el]]
      - Run ~M-x khoj <user-query>~ or Call ~C-c C-s~
 

tests/test_org_to_entries.py

@@ -393,6 +393,60 @@ def test_extract_entries_with_different_level_headings(tmp_path):
     assert entries[1][1].raw == "* Heading 2\n"
 
 
+def test_line_number_tracking_in_recursive_split():
+    "Ensure line numbers in URIs are correct after recursive splitting by checking against the actual file."
+    # Arrange
+    org_file_path = os.path.abspath("tests/data/org/main_readme.org")
+
+    with open(org_file_path, "r") as f:
+        org_content = f.read()
+    lines = org_content.splitlines()
+    data = {org_file_path: org_content}
+
+    # Act
+    # Using a small max_tokens to force recursive splitting
+    _, entries = OrgToEntries.extract_org_entries(org_files=data, max_tokens=10, index_heading_entries=True)
+
+    # Assert
+    assert len(entries) > 0, "No entries were extracted."
+
+    for entry in entries:
+        # Extract file path and line number from the entry URI
+        # for files uri is expected in format: file:///path/to/file.org#line=5
+        match = re.search(r"file://(.*?)#line=(\d+)", entry.uri)
+        if not match:
+            continue
+        filepath_from_uri = match.group(1)
+        line_number_from_uri = int(match.group(2))
+
+        # line_number is 1-based, list index is 0-based
+        line_in_file = clean(lines[line_number_from_uri - 1])
+        next_line_in_file = clean(lines[line_number_from_uri]) if line_number_from_uri < len(lines) else ""
+
+        # Remove ancestor heading lines inserted during post-processing
+        first_entry_line = ""
+        for line in entry.raw.splitlines():
+            if line.startswith("*"):
+                first_entry_line = line
+            else:
+                break  # Stop at the first non-heading line
+        # Remove heading prefix from entry.compiled as level changed during post-processing
+        cleaned_first_entry_line = first_entry_line.strip()
+        # Remove multiple consecutive spaces
+        cleaned_first_entry_line = clean(cleaned_first_entry_line)
+
+        assert entry.uri is not None, f"Entry '{entry}' has a None URI."
+        assert match is not None, f"URI format is incorrect: {entry.uri}"
+        assert (
+            filepath_from_uri == org_file_path
+        ), f"File path in URI '{filepath_from_uri}' does not match expected '{org_file_path}'"
+
+        # Ensure the first non-heading line in the compiled entry matches the line in the file
+        assert (
+            cleaned_first_entry_line in line_in_file.strip() or cleaned_first_entry_line in next_line_in_file.strip()
+        ), f"First non-heading line '{cleaned_first_entry_line}' in {entry.raw} does not match line {line_number_from_uri} in file: '{line_in_file}' or next line '{next_line_in_file}'"
+
+
 # Helper Functions
 def create_file(tmp_path, entry=None, filename="test.org"):
     org_file = tmp_path / filename
@@ -402,6 +456,6 @@ def create_file(tmp_path, entry=None, filename="test.org"):
     return org_file
 
 
-def clean(entry):
-    "Remove properties from entry for easier comparison."
-    return re.sub(r"\n:PROPERTIES:(.*?):END:", "", entry, flags=re.DOTALL)
+def clean(text):
+    "Normalize spaces in text for easier comparison."
+    return re.sub(r"\s+", " ", text)

tests/test_orgnode.py

@@ -100,9 +100,8 @@ def test_render_entry_with_property_drawer_and_empty_body(tmp_path):
 
     expected_entry = f"""*** [#A] Heading1                                            :tag1:
 :PROPERTIES:
-:LINE: file:{orgfile}::2
+:LINE: file://{orgfile}#line=2
 :ID: id:111-111-111-1111-1111
-:SOURCE: [[file:{orgfile}::*Heading1]]
 :END:
 """
 
@@ -133,37 +132,12 @@ Body Line 2
 
     # Assert
     # SOURCE link rendered with Heading
-    assert f":SOURCE: [[file:{orgfile}::*{entries[0].heading}]]" in f"{entries[0]}"
     # ID link rendered with ID
     assert f":ID: id:123-456-789-4234-1231" in f"{entries[0]}"
     # LINE link rendered with line number
-    assert f":LINE: file:{orgfile}::2" in f"{entries[0]}"
-
-
-# ----------------------------------------------------------------------------------------------------
-def test_source_link_to_entry_escaped_for_rendering(tmp_path):
-    "Test SOURCE link renders with square brackets in filename, heading escaped for org-mode rendering"
-    # Arrange
-    entry = f"""
-*** [#A] Heading[1]   :tag1:
-:PROPERTIES:
-:ID: 123-456-789-4234-1231
-:END:
-Body Line 1"""
-    orgfile = create_file(tmp_path, entry, filename="test[1].org")
-
-    # Act
-    entries = orgnode.makelist_with_filepath(orgfile)
-
-    # Assert
-    assert len(entries) == 1
-    # parsed heading from entry
-    assert entries[0].heading == "Heading[1]"
-    # track ancestors of entry
-    assert entries[0].ancestors == [f"{orgfile}"]
-    # ensure SOURCE link has square brackets in filename, heading escaped in rendered entries
-    escaped_orgfile = f"{orgfile}".replace("[1]", "\\[1\\]")
-    assert f":SOURCE: [[file:{escaped_orgfile}::*Heading\\[1\\]" in f"{entries[0]}"
+    assert f":LINE: file://{orgfile}#line=2" in f"{entries[0]}"
+    # LINE link rendered with line number
+    assert f":LINE: file://{orgfile}#line=7" in f"{entries[1]}"
 
 
 # ----------------------------------------------------------------------------------------------------