Create and use a context manager to time code

Use the timer context manager in all places where code was being timed

- Benefits
  - Deduplicate timing code scattered across codebase.
  - Provides single place to manage perf timing code
  - Use consistent timing log patterns

src/processor/ledger/beancount_to_jsonl.py

@@ -6,7 +6,7 @@ import time
 
 # Internal Packages
 from src.processor.text_to_jsonl import TextToJsonl
-from src.utils.helpers import get_absolute_path, is_none_or_empty
+from src.utils.helpers import get_absolute_path, is_none_or_empty, timer
 from src.utils.constants import empty_escape_sequences
 from src.utils.jsonl import dump_jsonl, compress_jsonl_data
 from src.utils.rawconfig import Entry
@@ -30,38 +30,30 @@ class BeancountToJsonl(TextToJsonl):
         beancount_files = BeancountToJsonl.get_beancount_files(beancount_files, beancount_file_filter)
 
         # Extract Entries from specified Beancount files
-        start = time.time()
-        current_entries = BeancountToJsonl.convert_transactions_to_maps(*BeancountToJsonl.extract_beancount_transactions(beancount_files))
-        end = time.time()
-        logger.debug(f"Parse transactions from Beancount files into dictionaries: {end - start} seconds")
+        with timer("Parse transactions from Beancount files into dictionaries", logger):
+            current_entries = BeancountToJsonl.convert_transactions_to_maps(*BeancountToJsonl.extract_beancount_transactions(beancount_files))
 
         # Split entries by max tokens supported by model
-        start = time.time()
-        current_entries = self.split_entries_by_max_tokens(current_entries, max_tokens=256)
-        end = time.time()
-        logger.debug(f"Split entries by max token size supported by model: {end - start} seconds")
+        with timer("Split entries by max token size supported by model", logger):
+            current_entries = self.split_entries_by_max_tokens(current_entries, max_tokens=256)
 
         # Identify, mark and merge any new entries with previous entries
-        start = time.time()
-        if not previous_entries:
-            entries_with_ids = list(enumerate(current_entries))
-        else:
-            entries_with_ids = self.mark_entries_for_update(current_entries, previous_entries, key='compiled', logger=logger)
-        end = time.time()
-        logger.debug(f"Identify new or updated transaction: {end - start} seconds")
+        with timer("Identify new or updated transaction", logger):
+            if not previous_entries:
+                entries_with_ids = list(enumerate(current_entries))
+            else:
+                entries_with_ids = self.mark_entries_for_update(current_entries, previous_entries, key='compiled', logger=logger)
 
-        # Process Each Entry from All Notes Files
-        start = time.time()
-        entries = list(map(lambda entry: entry[1], entries_with_ids))
-        jsonl_data = BeancountToJsonl.convert_transaction_maps_to_jsonl(entries)
+        with timer("Write transactions to JSONL file", logger):
+            # Process Each Entry from All Notes Files
+            entries = list(map(lambda entry: entry[1], entries_with_ids))
+            jsonl_data = BeancountToJsonl.convert_transaction_maps_to_jsonl(entries)
 
-        # Compress JSONL formatted Data
-        if output_file.suffix == ".gz":
-            compress_jsonl_data(jsonl_data, output_file)
-        elif output_file.suffix == ".jsonl":
-            dump_jsonl(jsonl_data, output_file)
-        end = time.time()
-        logger.debug(f"Write transactions to JSONL file: {end - start} seconds")
+            # Compress JSONL formatted Data
+            if output_file.suffix == ".gz":
+                compress_jsonl_data(jsonl_data, output_file)
+            elif output_file.suffix == ".jsonl":
+                dump_jsonl(jsonl_data, output_file)
 
         return entries_with_ids
 

src/processor/markdown/markdown_to_jsonl.py

@@ -6,7 +6,7 @@ import time
 
 # Internal Packages
 from src.processor.text_to_jsonl import TextToJsonl
-from src.utils.helpers import get_absolute_path, is_none_or_empty
+from src.utils.helpers import get_absolute_path, is_none_or_empty, timer
 from src.utils.constants import empty_escape_sequences
 from src.utils.jsonl import dump_jsonl, compress_jsonl_data
 from src.utils.rawconfig import Entry
@@ -30,38 +30,30 @@ class MarkdownToJsonl(TextToJsonl):
         markdown_files = MarkdownToJsonl.get_markdown_files(markdown_files, markdown_file_filter)
 
         # Extract Entries from specified Markdown files
-        start = time.time()
-        current_entries = MarkdownToJsonl.convert_markdown_entries_to_maps(*MarkdownToJsonl.extract_markdown_entries(markdown_files))
-        end = time.time()
-        logger.debug(f"Parse entries from Markdown files into dictionaries: {end - start} seconds")
+        with timer("Parse entries from Markdown files into dictionaries", logger):
+            current_entries = MarkdownToJsonl.convert_markdown_entries_to_maps(*MarkdownToJsonl.extract_markdown_entries(markdown_files))
 
         # Split entries by max tokens supported by model
-        start = time.time()
-        current_entries = self.split_entries_by_max_tokens(current_entries, max_tokens=256)
-        end = time.time()
-        logger.debug(f"Split entries by max token size supported by model: {end - start} seconds")
+        with timer("Split entries by max token size supported by model", logger):
+            current_entries = self.split_entries_by_max_tokens(current_entries, max_tokens=256)
 
         # Identify, mark and merge any new entries with previous entries
-        start = time.time()
-        if not previous_entries:
-            entries_with_ids = list(enumerate(current_entries))
-        else:
-            entries_with_ids = self.mark_entries_for_update(current_entries, previous_entries, key='compiled', logger=logger)
-        end = time.time()
-        logger.debug(f"Identify new or updated entries: {end - start} seconds")
+        with timer("Identify new or updated entries", logger):
+            if not previous_entries:
+                entries_with_ids = list(enumerate(current_entries))
+            else:
+                entries_with_ids = self.mark_entries_for_update(current_entries, previous_entries, key='compiled', logger=logger)
 
-        # Process Each Entry from All Notes Files
-        start = time.time()
-        entries = list(map(lambda entry: entry[1], entries_with_ids))
-        jsonl_data = MarkdownToJsonl.convert_markdown_maps_to_jsonl(entries)
+        with timer("Write markdown entries to JSONL file", logger):
+            # Process Each Entry from All Notes Files
+            entries = list(map(lambda entry: entry[1], entries_with_ids))
+            jsonl_data = MarkdownToJsonl.convert_markdown_maps_to_jsonl(entries)
 
-        # Compress JSONL formatted Data
-        if output_file.suffix == ".gz":
-            compress_jsonl_data(jsonl_data, output_file)
-        elif output_file.suffix == ".jsonl":
-            dump_jsonl(jsonl_data, output_file)
-        end = time.time()
-        logger.debug(f"Write markdown entries to JSONL file: {end - start} seconds")
+            # Compress JSONL formatted Data
+            if output_file.suffix == ".gz":
+                compress_jsonl_data(jsonl_data, output_file)
+            elif output_file.suffix == ".jsonl":
+                dump_jsonl(jsonl_data, output_file)
 
         return entries_with_ids
 

src/processor/org_mode/org_to_jsonl.py

@@ -7,7 +7,7 @@ from typing import Iterable
 # Internal Packages
 from src.processor.org_mode import orgnode
 from src.processor.text_to_jsonl import TextToJsonl
-from src.utils.helpers import get_absolute_path, is_none_or_empty
+from src.utils.helpers import get_absolute_path, is_none_or_empty, timer
 from src.utils.jsonl import dump_jsonl, compress_jsonl_data
 from src.utils.rawconfig import Entry
 from src.utils import state
@@ -29,24 +29,18 @@ class OrgToJsonl(TextToJsonl):
             exit(1)
 
         # Get Org Files to Process
-        start = time.time()
-        org_files = OrgToJsonl.get_org_files(org_files, org_file_filter)
+        with timer("Get org files to process", logger):
+            org_files = OrgToJsonl.get_org_files(org_files, org_file_filter)
 
         # Extract Entries from specified Org files
-        start = time.time()
-        entry_nodes, file_to_entries = self.extract_org_entries(org_files)
-        end = time.time()
-        logger.debug(f"Parse entries from org files into OrgNode objects: {end - start} seconds")
+        with timer("Parse entries from org files into OrgNode objects", logger):
+            entry_nodes, file_to_entries = self.extract_org_entries(org_files)
 
-        start = time.time()
-        current_entries = self.convert_org_nodes_to_entries(entry_nodes, file_to_entries, index_heading_entries)
-        end = time.time()
-        logger.debug(f"Convert OrgNodes into list of entries: {end - start} seconds")
+        with timer("Convert OrgNodes into list of entries", logger):
+            current_entries = self.convert_org_nodes_to_entries(entry_nodes, file_to_entries, index_heading_entries)
 
-        start = time.time()
-        current_entries = self.split_entries_by_max_tokens(current_entries, max_tokens=256)
-        end = time.time()
-        logger.debug(f"Split entries by max token size supported by model: {end - start} seconds")
+        with timer("Split entries by max token size supported by model", logger):
+            current_entries = self.split_entries_by_max_tokens(current_entries, max_tokens=256)
 
         # Identify, mark and merge any new entries with previous entries
         if not previous_entries:
@@ -55,17 +49,15 @@ class OrgToJsonl(TextToJsonl):
             entries_with_ids = self.mark_entries_for_update(current_entries, previous_entries, key='compiled', logger=logger)
 
         # Process Each Entry from All Notes Files
-        start = time.time()
-        entries = map(lambda entry: entry[1], entries_with_ids)
-        jsonl_data = self.convert_org_entries_to_jsonl(entries)
+        with timer("Write org entries to JSONL file", logger):
+            entries = map(lambda entry: entry[1], entries_with_ids)
+            jsonl_data = self.convert_org_entries_to_jsonl(entries)
 
-        # Compress JSONL formatted Data
-        if output_file.suffix == ".gz":
-            compress_jsonl_data(jsonl_data, output_file)
-        elif output_file.suffix == ".jsonl":
-            dump_jsonl(jsonl_data, output_file)
-        end = time.time()
-        logger.debug(f"Write org entries to JSONL file: {end - start} seconds")
+            # Compress JSONL formatted Data
+            if output_file.suffix == ".gz":
+                compress_jsonl_data(jsonl_data, output_file)
+            elif output_file.suffix == ".jsonl":
+                dump_jsonl(jsonl_data, output_file)
 
         return entries_with_ids
 

src/processor/text_to_jsonl.py

@@ -4,6 +4,7 @@ import hashlib
 import time
 import logging
 from typing import Callable
+from src.utils.helpers import timer
 
 # Internal Packages
 from src.utils.rawconfig import Entry, TextContentConfig
@@ -40,35 +41,31 @@ class TextToJsonl(ABC):
 
     def mark_entries_for_update(self, current_entries: list[Entry], previous_entries: list[Entry], key='compiled', logger=None) -> list[tuple[int, Entry]]:
         # Hash all current and previous entries to identify new entries
-        start = time.time()
-        current_entry_hashes = list(map(TextToJsonl.hash_func(key), current_entries))
-        previous_entry_hashes = list(map(TextToJsonl.hash_func(key), previous_entries))
-        end = time.time()
-        logger.debug(f"Hash previous, current entries: {end - start} seconds")
+        with timer("Hash previous, current entries", logger):
+            current_entry_hashes = list(map(TextToJsonl.hash_func(key), current_entries))
+            previous_entry_hashes = list(map(TextToJsonl.hash_func(key), previous_entries))
 
-        start = time.time()
-        hash_to_current_entries = dict(zip(current_entry_hashes, current_entries))
-        hash_to_previous_entries = dict(zip(previous_entry_hashes, previous_entries))
+        with timer("Identify, Mark, Combine new, existing entries", logger):
+            hash_to_current_entries = dict(zip(current_entry_hashes, current_entries))
+            hash_to_previous_entries = dict(zip(previous_entry_hashes, previous_entries))
 
-        # All entries that did not exist in the previous set are to be added
-        new_entry_hashes = set(current_entry_hashes) - set(previous_entry_hashes)
-        # All entries that exist in both current and previous sets are kept
-        existing_entry_hashes = set(current_entry_hashes) & set(previous_entry_hashes)
+            # All entries that did not exist in the previous set are to be added
+            new_entry_hashes = set(current_entry_hashes) - set(previous_entry_hashes)
+            # All entries that exist in both current and previous sets are kept
+            existing_entry_hashes = set(current_entry_hashes) & set(previous_entry_hashes)
 
-        # Mark new entries with -1 id to flag for later embeddings generation
-        new_entries = [
-            (-1, hash_to_current_entries[entry_hash])
-            for entry_hash in new_entry_hashes
-        ]
-        # Set id of existing entries to their previous ids to reuse their existing encoded embeddings
-        existing_entries = [
-            (previous_entry_hashes.index(entry_hash), hash_to_previous_entries[entry_hash])
-            for entry_hash in existing_entry_hashes
-        ]
+            # Mark new entries with -1 id to flag for later embeddings generation
+            new_entries = [
+                (-1, hash_to_current_entries[entry_hash])
+                for entry_hash in new_entry_hashes
+            ]
+            # Set id of existing entries to their previous ids to reuse their existing encoded embeddings
+            existing_entries = [
+                (previous_entry_hashes.index(entry_hash), hash_to_previous_entries[entry_hash])
+                for entry_hash in existing_entry_hashes
+            ]
 
-        existing_entries_sorted = sorted(existing_entries, key=lambda e: e[0])
-        entries_with_ids = existing_entries_sorted + new_entries
-        end = time.time()
-        logger.debug(f"Identify, Mark, Combine new, existing entries: {end - start} seconds")
+            existing_entries_sorted = sorted(existing_entries, key=lambda e: e[0])
+            entries_with_ids = existing_entries_sorted + new_entries
 
         return entries_with_ids