Disable ability to call <text>_to_jsonl, <type>_search packages directly

- This code is de-synced with expected args by above scripts
- Better to remove unused capabilitity that needlessly increases
  maintainance burden
This commit is contained in:
Debanjum Singh Solanky
2022-09-14 02:17:06 +03:00
parent 1680a617da
commit 0109c7bd91
5 changed files with 0 additions and 95 deletions

View File

@@ -128,16 +128,3 @@ def convert_transactions_to_maps(entries: list[str], transaction_to_file_map) ->
def convert_transaction_maps_to_jsonl(entries: list[dict]) -> str:
"Convert each Beancount transaction dictionary to JSON and collate as JSONL"
return ''.join([f'{json.dumps(entry_dict, ensure_ascii=False)}\n' for entry_dict in entries])
if __name__ == '__main__':
# Setup Argument Parser
parser = argparse.ArgumentParser(description="Map Beancount transactions into (compressed) JSONL format")
parser.add_argument('--output-file', '-o', type=pathlib.Path, required=True, help="Output file for (compressed) JSONL formatted transactions. Expected file extensions: jsonl or jsonl.gz")
parser.add_argument('--input-files', '-i', nargs='*', help="List of beancount files to process")
parser.add_argument('--input-filter', type=str, default=None, help="Regex filter for beancount files to process")
parser.add_argument('--verbose', '-v', action='count', default=0, help="Show verbose conversion logs, Default: 0")
args = parser.parse_args()
# Map transactions in beancount files to (compressed) JSONL formatted file
beancount_to_jsonl(args.input_files, args.input_filter, args.output_file, args.verbose)

View File

@@ -127,16 +127,3 @@ def convert_markdown_entries_to_maps(entries: list[str], entry_to_file_map) -> l
def convert_markdown_maps_to_jsonl(entries):
"Convert each Markdown entries to JSON and collate as JSONL"
return ''.join([f'{json.dumps(entry_dict, ensure_ascii=False)}\n' for entry_dict in entries])
if __name__ == '__main__':
# Setup Argument Parser
parser = argparse.ArgumentParser(description="Map Markdown entries into (compressed) JSONL format")
parser.add_argument('--output-file', '-o', type=pathlib.Path, required=True, help="Output file for (compressed) JSONL formatted notes. Expected file extensions: jsonl or jsonl.gz")
parser.add_argument('--input-files', '-i', nargs='*', help="List of markdown files to process")
parser.add_argument('--input-filter', type=str, default=None, help="Regex filter for markdown files to process")
parser.add_argument('--verbose', '-v', action='count', default=0, help="Show verbose conversion logs, Default: 0")
args = parser.parse_args()
# Map notes in Markdown files to (compressed) JSONL formatted file
markdown_to_jsonl(args.input_files, args.input_filter, args.output_file, args.verbose)

View File

@@ -153,16 +153,3 @@ def convert_org_nodes_to_entries(entries: list[orgnode.Orgnode], entry_to_file_m
def convert_org_entries_to_jsonl(entries: list[dict]) -> str:
"Convert each Org-Mode entry to JSON and collate as JSONL"
return ''.join([f'{json.dumps(entry_dict, ensure_ascii=False)}\n' for entry_dict in entries])
if __name__ == '__main__':
# Setup Argument Parser
parser = argparse.ArgumentParser(description="Map Org-Mode notes into (compressed) JSONL format")
parser.add_argument('--output-file', '-o', type=pathlib.Path, required=True, help="Output file for (compressed) JSONL formatted notes. Expected file extensions: jsonl or jsonl.gz")
parser.add_argument('--input-files', '-i', nargs='*', help="List of org-mode files to process")
parser.add_argument('--input-filter', type=str, default=None, help="Regex filter for org-mode files to process")
parser.add_argument('--verbose', '-v', action='count', default=0, help="Show verbose conversion logs, Default: 0")
args = parser.parse_args()
# Map notes in Org-Mode files to (compressed) JSONL formatted file
org_to_jsonl(args.input_files, args.input_filter, args.output_file, args.verbose)

View File

@@ -263,30 +263,3 @@ def setup(config: ImageContentConfig, search_config: ImageSearchConfig, regenera
image_embeddings,
image_metadata_embeddings,
encoder)
if __name__ == '__main__':
# Setup Argument Parser
parser = argparse.ArgumentParser(description="Semantic Search on Images")
parser.add_argument('--image-directory', '-i', required=True, type=pathlib.Path, help="Image directory to query")
parser.add_argument('--embeddings-file', '-e', default='image_embeddings.pt', type=pathlib.Path, help="File to save/load model embeddings to/from. Default: ./embeddings.pt")
parser.add_argument('--regenerate', action='store_true', default=False, help="Regenerate embeddings of Images in Image Directory . Default: false")
parser.add_argument('--results-count', '-n', default=5, type=int, help="Number of results to render. Default: 5")
parser.add_argument('--interactive', action='store_true', default=False, help="Interactive mode allows user to run queries on the model. Default: true")
parser.add_argument('--verbose', action='count', default=0, help="Show verbose conversion logs. Default: 0")
args = parser.parse_args()
image_names, image_embeddings, image_metadata_embeddings, model = setup(args.image_directory, args.embeddings_file, regenerate=args.regenerate)
# Run User Queries on Entries in Interactive Mode
while args.interactive:
# get query from user
user_query = input("Enter your query: ")
if user_query == "exit":
exit(0)
# query images
hits = query(user_query, image_embeddings, image_metadata_embeddings, model, args.results_count, args.verbose)
# render results
render_results(hits, image_names, args.image_directory, count=args.results_count)

View File

@@ -204,32 +204,3 @@ def setup(text_to_jsonl, config: TextContentConfig, search_config: TextSearchCon
filter.load(entries, regenerate=regenerate)
return TextSearchModel(entries, corpus_embeddings, bi_encoder, cross_encoder, filters, top_k)
if __name__ == '__main__':
# Setup Argument Parser
parser = argparse.ArgumentParser(description="Map Text files into (compressed) JSONL format")
parser.add_argument('--input-files', '-i', nargs='*', help="List of Text files to process")
parser.add_argument('--input-filter', type=str, default=None, help="Regex filter for Text files to process")
parser.add_argument('--compressed-jsonl', '-j', type=pathlib.Path, default=pathlib.Path("text.jsonl.gz"), help="Compressed JSONL to compute embeddings from")
parser.add_argument('--embeddings', '-e', type=pathlib.Path, default=pathlib.Path("text_embeddings.pt"), help="File to save/load model embeddings to/from")
parser.add_argument('--regenerate', action='store_true', default=False, help="Regenerate embeddings from text files. Default: false")
parser.add_argument('--results-count', '-n', default=5, type=int, help="Number of results to render. Default: 5")
parser.add_argument('--interactive', action='store_true', default=False, help="Interactive mode allows user to run queries on the model. Default: true")
parser.add_argument('--verbose', action='count', default=0, help="Show verbose conversion logs. Default: 0")
args = parser.parse_args()
entries, corpus_embeddings, bi_encoder, cross_encoder, top_k = setup(args.input_files, args.input_filter, args.compressed_jsonl, args.embeddings, args.regenerate)
# Run User Queries on Entries in Interactive Mode
while args.interactive:
# get query from user
user_query = input("Enter your query: ")
if user_query == "exit":
exit(0)
# query notes
hits = query(user_query, corpus_embeddings, entries, bi_encoder, cross_encoder, top_k)
# render results
render_results(hits, entries, count=args.results_count)