mirror of
https://github.com/khoaliber/khoj.git
synced 2026-03-09 13:25:11 +00:00
Use absolute file path to open files in org-to-jsonl.py, asymmetric.py
Exit script if neither org_files, org_file_filter is present
This commit is contained in:
@@ -12,7 +12,7 @@ import gzip
|
|||||||
# Define Functions
|
# Define Functions
|
||||||
def dump_jsonl(jsonl_data, output_path, verbose=0):
|
def dump_jsonl(jsonl_data, output_path, verbose=0):
|
||||||
"Write List of JSON objects to JSON line file"
|
"Write List of JSON objects to JSON line file"
|
||||||
with open(output_path, 'w', encoding='utf-8') as f:
|
with open(get_absolute_path(output_path), 'w', encoding='utf-8') as f:
|
||||||
f.write(jsonl_data)
|
f.write(jsonl_data)
|
||||||
|
|
||||||
if verbose > 0:
|
if verbose > 0:
|
||||||
@@ -30,7 +30,7 @@ def compress_jsonl_data(jsonl_data, output_path, verbose=0):
|
|||||||
def load_jsonl(input_path, verbose=0):
|
def load_jsonl(input_path, verbose=0):
|
||||||
"Read List of JSON objects from JSON line file"
|
"Read List of JSON objects from JSON line file"
|
||||||
data = []
|
data = []
|
||||||
with open(input_path, 'r', encoding='utf-8') as f:
|
with open(get_absolute_path(input_path), 'r', encoding='utf-8') as f:
|
||||||
for line in f:
|
for line in f:
|
||||||
data.append(json.loads(line.rstrip('\n|\r')))
|
data.append(json.loads(line.rstrip('\n|\r')))
|
||||||
|
|
||||||
@@ -105,7 +105,7 @@ def get_absolute_path(filepath):
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
# Setup argument parser
|
# Setup Argument Parser
|
||||||
parser = argparse.ArgumentParser(description="Map Org-Mode notes into (compressed) JSONL format")
|
parser = argparse.ArgumentParser(description="Map Org-Mode notes into (compressed) JSONL format")
|
||||||
parser.add_argument('--jsonl-file', '-o', type=pathlib.Path, required=True, help="Output file for JSONL formatted notes")
|
parser.add_argument('--jsonl-file', '-o', type=pathlib.Path, required=True, help="Output file for JSONL formatted notes")
|
||||||
parser.add_argument('--org-files', '-i', nargs='*', help="List of org-mode files to process")
|
parser.add_argument('--org-files', '-i', nargs='*', help="List of org-mode files to process")
|
||||||
@@ -116,6 +116,7 @@ if __name__ == '__main__':
|
|||||||
|
|
||||||
if is_none_or_empty(args.org_files) and is_none_or_empty(args.org_file_filter):
|
if is_none_or_empty(args.org_files) and is_none_or_empty(args.org_file_filter):
|
||||||
print("At least one of org-files or org-file-filter is required to be specified")
|
print("At least one of org-files or org-file-filter is required to be specified")
|
||||||
|
exit(1)
|
||||||
|
|
||||||
# Get Org Files to Process
|
# Get Org Files to Process
|
||||||
org_files = get_org_files(args.org_files, args.org_file_filter)
|
org_files = get_org_files(args.org_files, args.org_file_filter)
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ def initialize_model():
|
|||||||
def extract_entries(notesfile, verbose=False):
|
def extract_entries(notesfile, verbose=False):
|
||||||
"Load entries from compressed jsonl"
|
"Load entries from compressed jsonl"
|
||||||
entries = []
|
entries = []
|
||||||
with gzip.open(str(notesfile.expanduser()), 'rt', encoding='utf8') as jsonl:
|
with gzip.open(get_absolute_path(notesfile), 'rt', encoding='utf8') as jsonl:
|
||||||
for line in jsonl:
|
for line in jsonl:
|
||||||
note = json.loads(line.strip())
|
note = json.loads(line.strip())
|
||||||
|
|
||||||
@@ -44,13 +44,13 @@ def compute_embeddings(entries, bi_encoder, embeddings_file, verbose=False):
|
|||||||
"Compute (and Save) Embeddings or Load Pre-Computed Embeddings"
|
"Compute (and Save) Embeddings or Load Pre-Computed Embeddings"
|
||||||
# Load pre-computed embeddings from file if exists
|
# Load pre-computed embeddings from file if exists
|
||||||
if embeddings_file.exists():
|
if embeddings_file.exists():
|
||||||
corpus_embeddings = torch.load(str(embeddings_file.expanduser()))
|
corpus_embeddings = torch.load(get_absolute_path(embeddings_file))
|
||||||
if verbose:
|
if verbose:
|
||||||
print(f"Loaded embeddings from {embeddings_file}")
|
print(f"Loaded embeddings from {embeddings_file}")
|
||||||
|
|
||||||
else: # Else compute the corpus_embeddings from scratch, which can take a while
|
else: # Else compute the corpus_embeddings from scratch, which can take a while
|
||||||
corpus_embeddings = bi_encoder.encode(entries, convert_to_tensor=True, show_progress_bar=True)
|
corpus_embeddings = bi_encoder.encode(entries, convert_to_tensor=True, show_progress_bar=True)
|
||||||
torch.save(corpus_embeddings, str(embeddings_file.expanduser()))
|
torch.save(corpus_embeddings, get_absolute_path(embeddings_file))
|
||||||
if verbose:
|
if verbose:
|
||||||
print(f"Computed embeddings and save them to {embeddings_file}")
|
print(f"Computed embeddings and save them to {embeddings_file}")
|
||||||
|
|
||||||
@@ -140,6 +140,10 @@ def collate_results(hits, entries, count=5, verbose=False):
|
|||||||
in hits[0:count]]
|
in hits[0:count]]
|
||||||
|
|
||||||
|
|
||||||
|
def get_absolute_path(filepath):
|
||||||
|
return str(pathlib.Path(filepath).expanduser().absolute())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
# Setup Argument Parser
|
# Setup Argument Parser
|
||||||
parser = argparse.ArgumentParser(description="Map Org-Mode notes into JSONL format")
|
parser = argparse.ArgumentParser(description="Map Org-Mode notes into JSONL format")
|
||||||
|
|||||||
Reference in New Issue
Block a user