diff --git a/src/processor/ledger/beancount_to_jsonl.py b/src/processor/ledger/beancount_to_jsonl.py index 50fbda5d..b80fece6 100644 --- a/src/processor/ledger/beancount_to_jsonl.py +++ b/src/processor/ledger/beancount_to_jsonl.py @@ -128,16 +128,3 @@ def convert_transactions_to_maps(entries: list[str], transaction_to_file_map) -> def convert_transaction_maps_to_jsonl(entries: list[dict]) -> str: "Convert each Beancount transaction dictionary to JSON and collate as JSONL" return ''.join([f'{json.dumps(entry_dict, ensure_ascii=False)}\n' for entry_dict in entries]) - - -if __name__ == '__main__': - # Setup Argument Parser - parser = argparse.ArgumentParser(description="Map Beancount transactions into (compressed) JSONL format") - parser.add_argument('--output-file', '-o', type=pathlib.Path, required=True, help="Output file for (compressed) JSONL formatted transactions. Expected file extensions: jsonl or jsonl.gz") - parser.add_argument('--input-files', '-i', nargs='*', help="List of beancount files to process") - parser.add_argument('--input-filter', type=str, default=None, help="Regex filter for beancount files to process") - parser.add_argument('--verbose', '-v', action='count', default=0, help="Show verbose conversion logs, Default: 0") - args = parser.parse_args() - - # Map transactions in beancount files to (compressed) JSONL formatted file - beancount_to_jsonl(args.input_files, args.input_filter, args.output_file, args.verbose) diff --git a/src/processor/markdown/markdown_to_jsonl.py b/src/processor/markdown/markdown_to_jsonl.py index f0473700..80562a9f 100644 --- a/src/processor/markdown/markdown_to_jsonl.py +++ b/src/processor/markdown/markdown_to_jsonl.py @@ -127,16 +127,3 @@ def convert_markdown_entries_to_maps(entries: list[str], entry_to_file_map) -> l def convert_markdown_maps_to_jsonl(entries): "Convert each Markdown entries to JSON and collate as JSONL" return ''.join([f'{json.dumps(entry_dict, ensure_ascii=False)}\n' for entry_dict in entries]) - - -if __name__ == '__main__': - # Setup Argument Parser - parser = argparse.ArgumentParser(description="Map Markdown entries into (compressed) JSONL format") - parser.add_argument('--output-file', '-o', type=pathlib.Path, required=True, help="Output file for (compressed) JSONL formatted notes. Expected file extensions: jsonl or jsonl.gz") - parser.add_argument('--input-files', '-i', nargs='*', help="List of markdown files to process") - parser.add_argument('--input-filter', type=str, default=None, help="Regex filter for markdown files to process") - parser.add_argument('--verbose', '-v', action='count', default=0, help="Show verbose conversion logs, Default: 0") - args = parser.parse_args() - - # Map notes in Markdown files to (compressed) JSONL formatted file - markdown_to_jsonl(args.input_files, args.input_filter, args.output_file, args.verbose) diff --git a/src/processor/org_mode/org_to_jsonl.py b/src/processor/org_mode/org_to_jsonl.py index 52d70a68..699ceaa9 100644 --- a/src/processor/org_mode/org_to_jsonl.py +++ b/src/processor/org_mode/org_to_jsonl.py @@ -153,16 +153,3 @@ def convert_org_nodes_to_entries(entries: list[orgnode.Orgnode], entry_to_file_m def convert_org_entries_to_jsonl(entries: list[dict]) -> str: "Convert each Org-Mode entry to JSON and collate as JSONL" return ''.join([f'{json.dumps(entry_dict, ensure_ascii=False)}\n' for entry_dict in entries]) - - -if __name__ == '__main__': - # Setup Argument Parser - parser = argparse.ArgumentParser(description="Map Org-Mode notes into (compressed) JSONL format") - parser.add_argument('--output-file', '-o', type=pathlib.Path, required=True, help="Output file for (compressed) JSONL formatted notes. Expected file extensions: jsonl or jsonl.gz") - parser.add_argument('--input-files', '-i', nargs='*', help="List of org-mode files to process") - parser.add_argument('--input-filter', type=str, default=None, help="Regex filter for org-mode files to process") - parser.add_argument('--verbose', '-v', action='count', default=0, help="Show verbose conversion logs, Default: 0") - args = parser.parse_args() - - # Map notes in Org-Mode files to (compressed) JSONL formatted file - org_to_jsonl(args.input_files, args.input_filter, args.output_file, args.verbose) diff --git a/src/search_type/image_search.py b/src/search_type/image_search.py index bbef45a1..9ac8aa9a 100644 --- a/src/search_type/image_search.py +++ b/src/search_type/image_search.py @@ -263,30 +263,3 @@ def setup(config: ImageContentConfig, search_config: ImageSearchConfig, regenera image_embeddings, image_metadata_embeddings, encoder) - - -if __name__ == '__main__': - # Setup Argument Parser - parser = argparse.ArgumentParser(description="Semantic Search on Images") - parser.add_argument('--image-directory', '-i', required=True, type=pathlib.Path, help="Image directory to query") - parser.add_argument('--embeddings-file', '-e', default='image_embeddings.pt', type=pathlib.Path, help="File to save/load model embeddings to/from. Default: ./embeddings.pt") - parser.add_argument('--regenerate', action='store_true', default=False, help="Regenerate embeddings of Images in Image Directory . Default: false") - parser.add_argument('--results-count', '-n', default=5, type=int, help="Number of results to render. Default: 5") - parser.add_argument('--interactive', action='store_true', default=False, help="Interactive mode allows user to run queries on the model. Default: true") - parser.add_argument('--verbose', action='count', default=0, help="Show verbose conversion logs. Default: 0") - args = parser.parse_args() - - image_names, image_embeddings, image_metadata_embeddings, model = setup(args.image_directory, args.embeddings_file, regenerate=args.regenerate) - - # Run User Queries on Entries in Interactive Mode - while args.interactive: - # get query from user - user_query = input("Enter your query: ") - if user_query == "exit": - exit(0) - - # query images - hits = query(user_query, image_embeddings, image_metadata_embeddings, model, args.results_count, args.verbose) - - # render results - render_results(hits, image_names, args.image_directory, count=args.results_count) diff --git a/src/search_type/text_search.py b/src/search_type/text_search.py index 90c822ac..922fab1a 100644 --- a/src/search_type/text_search.py +++ b/src/search_type/text_search.py @@ -204,32 +204,3 @@ def setup(text_to_jsonl, config: TextContentConfig, search_config: TextSearchCon filter.load(entries, regenerate=regenerate) return TextSearchModel(entries, corpus_embeddings, bi_encoder, cross_encoder, filters, top_k) - - -if __name__ == '__main__': - # Setup Argument Parser - parser = argparse.ArgumentParser(description="Map Text files into (compressed) JSONL format") - parser.add_argument('--input-files', '-i', nargs='*', help="List of Text files to process") - parser.add_argument('--input-filter', type=str, default=None, help="Regex filter for Text files to process") - parser.add_argument('--compressed-jsonl', '-j', type=pathlib.Path, default=pathlib.Path("text.jsonl.gz"), help="Compressed JSONL to compute embeddings from") - parser.add_argument('--embeddings', '-e', type=pathlib.Path, default=pathlib.Path("text_embeddings.pt"), help="File to save/load model embeddings to/from") - parser.add_argument('--regenerate', action='store_true', default=False, help="Regenerate embeddings from text files. Default: false") - parser.add_argument('--results-count', '-n', default=5, type=int, help="Number of results to render. Default: 5") - parser.add_argument('--interactive', action='store_true', default=False, help="Interactive mode allows user to run queries on the model. Default: true") - parser.add_argument('--verbose', action='count', default=0, help="Show verbose conversion logs. Default: 0") - args = parser.parse_args() - - entries, corpus_embeddings, bi_encoder, cross_encoder, top_k = setup(args.input_files, args.input_filter, args.compressed_jsonl, args.embeddings, args.regenerate) - - # Run User Queries on Entries in Interactive Mode - while args.interactive: - # get query from user - user_query = input("Enter your query: ") - if user_query == "exit": - exit(0) - - # query notes - hits = query(user_query, corpus_embeddings, entries, bi_encoder, cross_encoder, top_k) - - # render results - render_results(hits, entries, count=args.results_count)