feat: define search folder per newsletter #3

backend/alembic/versions/75ed3dbf1e16_add_search_folder_to_newsletter.py

@@ -0,0 +1,32 @@
+"""add search_folder to newsletter
+
+Revision ID: 75ed3dbf1e16
+Revises: 1ed7baeaf282
+Create Date: 2025-08-12 21:13:07.535625
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '75ed3dbf1e16'
+down_revision: Union[str, Sequence[str], None] = '1ed7baeaf282'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('newsletters', sa.Column('search_folder', sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('newsletters', 'search_folder')
+    # ### end Alembic commands ###

backend/app/crud/newsletters.py

@@ -64,6 +64,7 @@ def create_newsletter(db: Session, newsletter: NewsletterCreate):
         id=generate(size=10),
         name=newsletter.name,
         slug=newsletter.slug,
+        search_folder=newsletter.search_folder,
         extract_content=newsletter.extract_content,
         move_to_folder=newsletter.move_to_folder,
     )

backend/app/models/newsletters.py

@@ -12,6 +12,7 @@ class Newsletter(Base):
     id = Column(String, primary_key=True, index=True)
     slug = Column(String, unique=True, index=True, nullable=True)
     name = Column(String)
+    search_folder = Column(String, nullable=True)
     move_to_folder = Column(String, nullable=True)
     is_active = Column(Boolean, default=True)
     extract_content = Column(Boolean, default=False)

backend/app/schemas/newsletters.py

@@ -31,6 +31,7 @@ class NewsletterBase(BaseModel):
 
     name: str
     slug: str | None = None
+    search_folder: str | None = None
     move_to_folder: str | None = None
     extract_content: bool = False
 

backend/app/services/email_processor.py

@@ -34,20 +34,22 @@ def _is_configured(settings: Settings | None) -> bool:
     return True
 
 
-def _connect_to_imap(settings: Settings) -> imaplib.IMAP4_SSL | None:
+def _connect_to_imap(
+    settings: Settings, search_folder: str
+) -> imaplib.IMAP4_SSL | None:
     """Connect to the IMAP server and select the mailbox."""
     try:
         logger.info(f"Connecting to IMAP server: {settings.imap_server}")
         mail = imaplib.IMAP4_SSL(settings.imap_server)
         mail.login(settings.imap_username, settings.imap_password)
-        status, messages = mail.select(settings.search_folder)
+        status, messages = mail.select(search_folder)
         if status != "OK":
             logger.error(
-                f"Failed to select mailbox: {settings.search_folder}, status: {status}, messages: {messages}"
+                f"Failed to select mailbox: {search_folder}, status: {status}, messages: {messages}"
             )
             mail.logout()
             return None
-        logger.info(f"Selected mailbox: {settings.search_folder}")
+        logger.info(f"Selected mailbox: {search_folder}")
         return mail
     except Exception as e:
         logger.error(f"Failed to connect to IMAP server: {e}", exc_info=True)
@@ -235,26 +237,61 @@ def process_emails(db: Session) -> None:
     if not _is_configured(settings):
         return
 
-    newsletters = get_newsletters(db)
-    sender_map = {sender.email: nl for nl in newsletters for sender in nl.senders}
-    logger.info(f"Processing emails for {len(newsletters)} newsletters.")
+    all_newsletters = get_newsletters(db)
+    logger.info(f"Processing emails for {len(all_newsletters)} newsletters.")
 
-    mail = _connect_to_imap(settings)
-    if not mail:
-        return
+    # Group newsletters by search folder
+    folder_groups: dict[str, list[Newsletter]] = {}
+    for nl in all_newsletters:
+        folder = nl.search_folder or settings.search_folder
+        if folder not in folder_groups:
+            folder_groups[folder] = []
+        folder_groups[folder].append(nl)
 
-    try:
-        email_ids = _fetch_unread_email_ids(mail)
-        logger.info(f"Found {len(email_ids)} unseen emails.")
-        for num in email_ids:
-            _process_single_email(num, mail, db, sender_map, settings)
+    # If auto-adding is enabled, ensure the default search folder is always checked.
+    if settings.auto_add_new_senders and settings.search_folder not in folder_groups:
+        folder_groups[settings.search_folder] = []
 
-        if settings.move_to_folder:
-            logger.info("Expunging deleted emails")
-            mail.expunge()
+    for search_folder, newsletters_in_folder in folder_groups.items():
+        logger.info(
+            f"Processing folder '{search_folder}' for {len(newsletters_in_folder)} newsletters."
+        )
+        sender_map = {
+            sender.email: nl for nl in newsletters_in_folder for sender in nl.senders
+        }
 
-    except Exception as e:
-        logger.error(f"Error processing emails: {e}", exc_info=True)
-    finally:
-        mail.logout()
-        logger.info("Email processing finished successfully.")
+        mail = _connect_to_imap(settings, search_folder)
+        if not mail:
+            logger.warning(
+                f"Skipping folder '{search_folder}' due to connection issue."
+            )
+            continue
+
+        try:
+            email_ids = _fetch_unread_email_ids(mail)
+            logger.info(
+                f"Found {len(email_ids)} unseen emails in folder '{search_folder}'."
+            )
+            for num in email_ids:
+                _process_single_email(num, mail, db, sender_map, settings)
+
+            # Expunge logic needs to be carefully considered.
+            # If any newsletter in this folder group has a move_to_folder, we expunge.
+            # This is an approximation. A more robust solution might require per-email expunge.
+            should_expunge = any(
+                nl.move_to_folder or settings.move_to_folder
+                for nl in newsletters_in_folder
+            )
+            if should_expunge:
+                logger.info(f"Expunging deleted emails from '{search_folder}'")
+                mail.expunge()
+
+        except Exception as e:
+            logger.error(
+                f"Error processing emails in folder '{search_folder}': {e}",
+                exc_info=True,
+            )
+        finally:
+            mail.logout()
+
+    logger.info("Email processing finished successfully.")

backend/app/tests/test_crud.py

@@ -152,6 +152,33 @@ def test_create_newsletter_with_move_to_folder(db_session: Session):
     assert retrieved_newsletter.extract_content is True
 
 
+def test_create_newsletter_with_search_folder(db_session: Session):
+    """Test creating and updating a newsletter with the search_folder attribute."""
+    unique_email = f"sender_{uuid.uuid4()}@test.com"
+    newsletter_data = NewsletterCreate(
+        name="Test Newsletter with Search Folder",
+        sender_emails=[unique_email],
+        search_folder="CustomInbox",
+    )
+    newsletter = create_newsletter(db_session, newsletter_data)
+    retrieved_newsletter = get_newsletter_by_identifier(db_session, newsletter.id)
+
+    assert retrieved_newsletter.name == "Test Newsletter with Search Folder"
+    assert retrieved_newsletter.search_folder == "CustomInbox"
+
+    # Test updating the search_folder
+    from app.crud.newsletters import update_newsletter
+    from app.schemas.newsletters import NewsletterUpdate
+
+    update_data = NewsletterUpdate(
+        name=newsletter.name,
+        sender_emails=[unique_email],
+        search_folder="UpdatedCustomInbox",
+    )
+    updated_newsletter = update_newsletter(db_session, newsletter.id, update_data)
+    assert updated_newsletter.search_folder == "UpdatedCustomInbox"
+
+
 def test_get_newsletter_by_identifier(db_session: Session):
     """Test getting a single newsletter."""
     unique_email = f"sender_{uuid.uuid4()}@test.com"

backend/app/tests/test_email_processor.py

@@ -8,15 +8,15 @@ from app.crud.newsletters import create_newsletter
 from app.crud.settings import create_or_update_settings
 from app.models.newsletters import Newsletter
 from app.schemas.newsletters import NewsletterCreate
-from app.schemas.settings import SettingsCreate
-from app.services.email_processor import _process_single_email
+from app.schemas.settings import Settings, SettingsCreate
+from app.services.email_processor import _process_single_email, process_emails
 
 
 def _setup_test_email_processing(
     db_session: Session,
     newsletter_create_data: NewsletterCreate,
     settings_create_data: SettingsCreate,
-) -> tuple[MagicMock, Newsletter, SettingsCreate]:
+) -> tuple[MagicMock, Newsletter, Settings]:
     """Help to set up mocks and data for email processing tests."""
     settings = create_or_update_settings(db_session, settings_create_data)
     newsletter = create_newsletter(db_session, newsletter_create_data)
@@ -84,6 +84,74 @@ def test_process_single_email_with_global_move_folder(db_session: Session):
     mock_mail.store.assert_any_call("1", "+FLAGS", "\\Deleted")
 
 
+@patch("app.services.email_processor._connect_to_imap")
+def test_process_emails_uses_newsletter_search_folder(
+    mock_connect_to_imap,
+    db_session: Session,
+):
+    """Test that the per-newsletter search_folder is used, overriding the global setting."""
+    # 1. ARRANGE
+    settings_data = SettingsCreate(
+        imap_server="test.com",
+        imap_username="test",
+        imap_password="password",
+        search_folder="GlobalInbox",
+    )
+    create_or_update_settings(db_session, settings_data)
+
+    newsletter_data = NewsletterCreate(
+        name="Test Newsletter",
+        sender_emails=["test@example.com"],
+        search_folder="NewsletterInbox",
+    )
+    create_newsletter(db_session, newsletter_data)
+
+    # Mock the return of _connect_to_imap to avoid a real IMAP connection
+    mock_connect_to_imap.return_value = None
+
+    # 2. ACT
+    process_emails(db_session)
+
+    # 3. ASSERT
+    # Check that _connect_to_imap was called with the newsletter's specific folder
+    mock_connect_to_imap.assert_called_once()
+    call_args = mock_connect_to_imap.call_args[0]
+    assert call_args[1] == "NewsletterInbox"
+
+
+@patch("app.services.email_processor._connect_to_imap")
+def test_process_emails_uses_global_search_folder(
+    mock_connect_to_imap,
+    db_session: Session,
+):
+    """Test that the global search_folder is used when the per-newsletter one is not set."""
+    # 1. ARRANGE
+    settings_data = SettingsCreate(
+        imap_server="test.com",
+        imap_username="test",
+        imap_password="password",
+        search_folder="GlobalInbox",
+    )
+    create_or_update_settings(db_session, settings_data)
+
+    newsletter_data = NewsletterCreate(
+        name="Test Newsletter",
+        sender_emails=["test@example.com"],
+        search_folder=None,  # Explicitly not set
+    )
+    create_newsletter(db_session, newsletter_data)
+
+    mock_connect_to_imap.return_value = None
+
+    # 2. ACT
+    process_emails(db_session)
+
+    # 3. ASSERT
+    mock_connect_to_imap.assert_called_once()
+    call_args = mock_connect_to_imap.call_args[0]
+    assert call_args[1] == "GlobalInbox"
+
+
 @patch("app.services.email_processor._extract_and_clean_html")
 def test_process_single_email_with_content_extraction(
     mock_extract_clean,