klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-04 21:29:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Consolidate jsonl helper methods in a single file under utils module

Browse Source
This commit is contained in:
Debanjum Singh Solanky
2022-07-21 03:28:37 +04:00
parent de726c4b6c
commit 0917f1574d
6 changed files with 57 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/processor/ledger/beancount_to_jsonl.py
View File

@@ -5,12 +5,12 @@ import json
import argparse
import pathlib
import glob
import gzip
import re
# Internal Packages
from src.utils.helpers import get_absolute_path, is_none_or_empty
from src.utils.constants import empty_escape_sequences
from src.utils.jsonl import dump_jsonl, compress_jsonl_data
# Define Functions
@@ -38,25 +38,6 @@ def beancount_to_jsonl(beancount_files, beancount_file_filter, output_file, verb
return entries
def dump_jsonl(jsonl_data, output_path, verbose=0):
"Write List of JSON objects to JSON line file"
with open(get_absolute_path(output_path), 'w', encoding='utf-8') as f:
f.write(jsonl_data)
if verbose > 0:
jsonl_entries = len(jsonl_data.split('\n'))
print(f'Wrote {jsonl_entries} lines to jsonl at {output_path}')
def compress_jsonl_data(jsonl_data, output_path, verbose=0):
with gzip.open(get_absolute_path(output_path), 'wt') as gzip_file:
gzip_file.write(jsonl_data)
if verbose > 0:
jsonl_entries = len(jsonl_data.split('\n'))
print(f'Wrote {jsonl_entries} lines to gzip compressed jsonl at {output_path}')
def get_beancount_files(beancount_files=None, beancount_file_filter=None, verbose=0):
"Get Beancount files to process"
absolute_beancount_files, filtered_beancount_files = set(), set()