klbr/LetterFeed
1
0
Fork 0
mirror of https://github.com/khoaliber/LetterFeed.git synced 2026-03-02 05:29:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13 from LeonMusCoden/feat/custom-search-folder

Browse Source 
feat: define search folder per newsletter #3
This commit is contained in:
Leon
2025-08-12 19:13:27 -04:00
committed by GitHub
parent eefd5b4050 8fa474738a
commit 22e8eeac3d
10 changed files with 224 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
backend/alembic/versions/75ed3dbf1e16_add_search_folder_to_newsletter.py Normal file
View File

@@ -0,0 +1,32 @@
"""add search_folder to newsletter
Revision ID: 75ed3dbf1e16
Revises: 1ed7baeaf282
Create Date: 2025-08-12 21:13:07.535625
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '75ed3dbf1e16'
down_revision: Union[str, Sequence[str], None] = '1ed7baeaf282'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('newsletters', sa.Column('search_folder', sa.String(), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('newsletters', 'search_folder')
# ### end Alembic commands ###

1
backend/app/crud/newsletters.py
View File

@@ -64,6 +64,7 @@ def create_newsletter(db: Session, newsletter: NewsletterCreate):
id=generate(size=10),
name=newsletter.name,
slug=newsletter.slug,
search_folder=newsletter.search_folder,
extract_content=newsletter.extract_content,
move_to_folder=newsletter.move_to_folder,
)

1
backend/app/models/newsletters.py
View File

@@ -12,6 +12,7 @@ class Newsletter(Base):
id = Column(String, primary_key=True, index=True)
slug = Column(String, unique=True, index=True, nullable=True)
name = Column(String)
search_folder = Column(String, nullable=True)
move_to_folder = Column(String, nullable=True)
is_active = Column(Boolean, default=True)
extract_content = Column(Boolean, default=False)

1
backend/app/schemas/newsletters.py
View File

@@ -31,6 +31,7 @@ class NewsletterBase(BaseModel):
name: str
slug: str | None = None
search_folder: str | None = None
move_to_folder: str | None = None
extract_content: bool = False

83
backend/app/services/email_processor.py
View File

@@ -34,20 +34,22 @@ def _is_configured(settings: Settings | None) -> bool:
return True
def _connect_to_imap(settings: Settings) -> imaplib.IMAP4_SSL | None:
def _connect_to_imap(
settings: Settings, search_folder: str
) -> imaplib.IMAP4_SSL | None:
"""Connect to the IMAP server and select the mailbox."""
try:
logger.info(f"Connecting to IMAP server: {settings.imap_server}")
mail = imaplib.IMAP4_SSL(settings.imap_server)
mail.login(settings.imap_username, settings.imap_password)
status, messages = mail.select(settings.search_folder)
status, messages = mail.select(search_folder)
if status != "OK":
logger.error(
f"Failed to select mailbox: {settings.search_folder}, status: {status}, messages: {messages}"
f"Failed to select mailbox: {search_folder}, status: {status}, messages: {messages}"
)
mail.logout()
return None
logger.info(f"Selected mailbox: {settings.search_folder}")
logger.info(f"Selected mailbox: {search_folder}")
return mail
except Exception as e:
logger.error(f"Failed to connect to IMAP server: {e}", exc_info=True)
@@ -235,26 +237,61 @@ def process_emails(db: Session) -> None:
if not _is_configured(settings):
return
newsletters = get_newsletters(db)
sender_map = {sender.email: nl for nl in newsletters for sender in nl.senders}
logger.info(f"Processing emails for {len(newsletters)} newsletters.")
all_newsletters = get_newsletters(db)
logger.info(f"Processing emails for {len(all_newsletters)} newsletters.")
mail = _connect_to_imap(settings)
if not mail:
return
# Group newsletters by search folder
folder_groups: dict[str, list[Newsletter]] = {}
for nl in all_newsletters:
folder = nl.search_folder or settings.search_folder
if folder not in folder_groups:
folder_groups[folder] = []
folder_groups[folder].append(nl)
try:
email_ids = _fetch_unread_email_ids(mail)
logger.info(f"Found {len(email_ids)} unseen emails.")
for num in email_ids:
_process_single_email(num, mail, db, sender_map, settings)
# If auto-adding is enabled, ensure the default search folder is always checked.
if settings.auto_add_new_senders and settings.search_folder not in folder_groups:
folder_groups[settings.search_folder] = []
if settings.move_to_folder:
logger.info("Expunging deleted emails")
mail.expunge()
for search_folder, newsletters_in_folder in folder_groups.items():
logger.info(
f"Processing folder '{search_folder}' for {len(newsletters_in_folder)} newsletters."
)
sender_map = {
sender.email: nl for nl in newsletters_in_folder for sender in nl.senders
}
except Exception as e:
logger.error(f"Error processing emails: {e}", exc_info=True)
finally:
mail.logout()
logger.info("Email processing finished successfully.")
mail = _connect_to_imap(settings, search_folder)
if not mail:
logger.warning(
f"Skipping folder '{search_folder}' due to connection issue."
)
continue
try:
email_ids = _fetch_unread_email_ids(mail)
logger.info(
f"Found {len(email_ids)} unseen emails in folder '{search_folder}'."
)
for num in email_ids:
_process_single_email(num, mail, db, sender_map, settings)
# Expunge logic needs to be carefully considered.
# If any newsletter in this folder group has a move_to_folder, we expunge.
# This is an approximation. A more robust solution might require per-email expunge.
should_expunge = any(
nl.move_to_folder or settings.move_to_folder
for nl in newsletters_in_folder
)
if should_expunge:
logger.info(f"Expunging deleted emails from '{search_folder}'")
mail.expunge()
except Exception as e:
logger.error(
f"Error processing emails in folder '{search_folder}': {e}",
exc_info=True,
)
finally:
mail.logout()
logger.info("Email processing finished successfully.")

27
backend/app/tests/test_crud.py
View File

@@ -152,6 +152,33 @@ def test_create_newsletter_with_move_to_folder(db_session: Session):
assert retrieved_newsletter.extract_content is True
def test_create_newsletter_with_search_folder(db_session: Session):
"""Test creating and updating a newsletter with the search_folder attribute."""
unique_email = f"sender_{uuid.uuid4()}@test.com"
newsletter_data = NewsletterCreate(
name="Test Newsletter with Search Folder",
sender_emails=[unique_email],
search_folder="CustomInbox",
)
newsletter = create_newsletter(db_session, newsletter_data)
retrieved_newsletter = get_newsletter_by_identifier(db_session, newsletter.id)
assert retrieved_newsletter.name == "Test Newsletter with Search Folder"
assert retrieved_newsletter.search_folder == "CustomInbox"
# Test updating the search_folder
from app.crud.newsletters import update_newsletter
from app.schemas.newsletters import NewsletterUpdate
update_data = NewsletterUpdate(
name=newsletter.name,
sender_emails=[unique_email],
search_folder="UpdatedCustomInbox",
)
updated_newsletter = update_newsletter(db_session, newsletter.id, update_data)
assert updated_newsletter.search_folder == "UpdatedCustomInbox"
def test_get_newsletter_by_identifier(db_session: Session):
"""Test getting a single newsletter."""
unique_email = f"sender_{uuid.uuid4()}@test.com"

74
backend/app/tests/test_email_processor.py
View File

@@ -8,15 +8,15 @@ from app.crud.newsletters import create_newsletter
from app.crud.settings import create_or_update_settings
from app.models.newsletters import Newsletter
from app.schemas.newsletters import NewsletterCreate
from app.schemas.settings import SettingsCreate
from app.services.email_processor import _process_single_email
from app.schemas.settings import Settings, SettingsCreate
from app.services.email_processor import _process_single_email, process_emails
def _setup_test_email_processing(
db_session: Session,
newsletter_create_data: NewsletterCreate,
settings_create_data: SettingsCreate,
) -> tuple[MagicMock, Newsletter, SettingsCreate]:
) -> tuple[MagicMock, Newsletter, Settings]:
"""Help to set up mocks and data for email processing tests."""
settings = create_or_update_settings(db_session, settings_create_data)
newsletter = create_newsletter(db_session, newsletter_create_data)
@@ -84,6 +84,74 @@ def test_process_single_email_with_global_move_folder(db_session: Session):
mock_mail.store.assert_any_call("1", "+FLAGS", "\\Deleted")
@patch("app.services.email_processor._connect_to_imap")
def test_process_emails_uses_newsletter_search_folder(
mock_connect_to_imap,
db_session: Session,
):
"""Test that the per-newsletter search_folder is used, overriding the global setting."""
# 1. ARRANGE
settings_data = SettingsCreate(
imap_server="test.com",
imap_username="test",
imap_password="password",
search_folder="GlobalInbox",
)
create_or_update_settings(db_session, settings_data)
newsletter_data = NewsletterCreate(
name="Test Newsletter",
sender_emails=["test@example.com"],
search_folder="NewsletterInbox",
)
create_newsletter(db_session, newsletter_data)
# Mock the return of _connect_to_imap to avoid a real IMAP connection
mock_connect_to_imap.return_value = None
# 2. ACT
process_emails(db_session)
# 3. ASSERT
# Check that _connect_to_imap was called with the newsletter's specific folder
mock_connect_to_imap.assert_called_once()
call_args = mock_connect_to_imap.call_args[0]
assert call_args[1] == "NewsletterInbox"
@patch("app.services.email_processor._connect_to_imap")
def test_process_emails_uses_global_search_folder(
mock_connect_to_imap,
db_session: Session,
):
"""Test that the global search_folder is used when the per-newsletter one is not set."""
# 1. ARRANGE
settings_data = SettingsCreate(
imap_server="test.com",
imap_username="test",
imap_password="password",
search_folder="GlobalInbox",
)
create_or_update_settings(db_session, settings_data)
newsletter_data = NewsletterCreate(
name="Test Newsletter",
sender_emails=["test@example.com"],
search_folder=None, # Explicitly not set
)
create_newsletter(db_session, newsletter_data)
mock_connect_to_imap.return_value = None
# 2. ACT
process_emails(db_session)
# 3. ASSERT
mock_connect_to_imap.assert_called_once()
call_args = mock_connect_to_imap.call_args[0]
assert call_args[1] == "GlobalInbox"
@patch("app.services.email_processor._extract_and_clean_html")
def test_process_single_email_with_content_extraction(
mock_extract_clean,

25
frontend/src/components/letterfeed/NewsletterDialog.tsx
View File

@@ -36,6 +36,7 @@ const getInitialState = (newsletter: Newsletter | null | undefined) => {
name: newsletter.name,
slug: newsletter.slug || "",
emails: newsletter.senders.map((s) => s.email),
search_folder: newsletter.search_folder || "",
move_to_folder: newsletter.move_to_folder || "",
extract_content: newsletter.extract_content,
}
@@ -44,6 +45,7 @@ const getInitialState = (newsletter: Newsletter | null | undefined) => {
name: "",
slug: "",
emails: [""],
search_folder: "",
move_to_folder: "",
extract_content: false,
}
@@ -89,6 +91,7 @@ export function NewsletterDialog({ newsletter, isOpen, folderOptions, onOpenChan
name: formData.name,
slug: formData.slug,
sender_emails: formData.emails.filter((email) => email.trim()),
search_folder: formData.search_folder,
move_to_folder: formData.move_to_folder,
extract_content: formData.extract_content,
}
@@ -148,6 +151,28 @@ export function NewsletterDialog({ newsletter, isOpen, folderOptions, onOpenChan
/>
</div>
<div className="space-y-2">
<Label htmlFor="search_folder">Folder to Search</Label>
<Select
value={formData.search_folder || "None"}
onValueChange={(value) =>
setFormData((prev) => ({ ...prev, search_folder: value === "None" ? "" : value }))
}
>
<SelectTrigger>
<SelectValue placeholder="Select folder or leave empty" />
</SelectTrigger>
<SelectContent>
<SelectItem value="None">Default (use global setting)</SelectItem>
{folderOptions.map((folder) => (
<SelectItem key={folder} value={folder}>
{folder}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
<div className="space-y-2">
<Label htmlFor="move_to_folder">Move To Folder</Label>
<Select

3
frontend/src/components/letterfeed/__tests__/NewsletterDialog.test.tsx
View File

@@ -23,6 +23,7 @@ const mockNewsletter: Newsletter = {
extract_content: false,
senders: [{ id: "1", email: "current@example.com" }],
entries_count: 5,
search_folder: "",
move_to_folder: "",
}
@@ -62,6 +63,7 @@ describe("NewsletterDialog", () => {
name: "My New Newsletter",
slug: "my-new-newsletter",
sender_emails: ["test@example.com"],
search_folder: "",
move_to_folder: "",
extract_content: false,
})
@@ -109,6 +111,7 @@ describe("NewsletterDialog", () => {
name: "Updated Name",
slug: "existing-newsletter",
sender_emails: ["current@example.com"],
search_folder: "",
move_to_folder: "",
extract_content: false,
})

3
frontend/src/lib/api.ts
View File

@@ -11,6 +11,7 @@ export interface Newsletter {
name: string
slug: string | null
is_active: boolean
search_folder?: string | null
move_to_folder?: string | null
extract_content: boolean
senders: { id: string; email: string }[]
@@ -21,6 +22,7 @@ export interface NewsletterCreate {
name: string;
slug?: string | null;
sender_emails: string[];
search_folder?: string | null;
move_to_folder?: string | null;
extract_content: boolean;
}
@@ -29,6 +31,7 @@ export interface NewsletterUpdate {
name: string;
slug?: string | null;
sender_emails: string[];
search_folder?: string | null;
move_to_folder?: string | null;
extract_content: boolean;
}