#!/usr/bin/env python3 """Maildir-Volltextindex (SQLite FTS5) fuer den MCP-Mail-Server. Hintergrund: search_mail scannte bei jeder Anfrage das komplette Maildir (Roh-Byte-Vorfilter ueber alle Dateien). Bei Katis Gmail-Archiv (16 GB, ~30k Mails) dauerte das laenger als das Gateway-Request-Timeout → "Request timed out". Dieser Index macht Suchen zu FTS5-SQL-Queries (Millisekunden). Design: - Eine SQLite-DB pro User unter /var/lib/mail-index/.sqlite - Tabelle messages (Stammdaten + body) + virtuelle Tabelle messages_fts (FTS5, content='messages', mit Trigger-Sync via externem Content) - Inkrementell: Datei-Pfad + mtime + size in messages; beim Update nur neue/geaenderte Dateien neu einlesen, entfernte loeschen - Such-Strategie: FTS5-MATCH mit Prefix-Termen; Fallback auf LIKE ueber sender/to/subject (deckt Substrings und E-Mail-Adressen ab), zuletzt LIKE ueber body """ import os import re import sqlite3 import sys import time import html as htmlmod from email import policy from email.parser import BytesParser from email.utils import parsedate_to_datetime BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, BASE_DIR) from common import load_config as _lc # noqa: E402 INDEX_DIR = "/var/lib/mail-index" _SCHEMA = """ CREATE TABLE IF NOT EXISTS messages ( id INTEGER PRIMARY KEY, path TEXT UNIQUE NOT NULL, account TEXT NOT NULL, folder TEXT NOT NULL, key TEXT NOT NULL, mtime REAL NOT NULL, size INTEGER NOT NULL, date_ts INTEGER DEFAULT 0, sent_from TEXT DEFAULT '', sent_to TEXT DEFAULT '', subject TEXT DEFAULT '', body TEXT DEFAULT '' ); CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( sent_from, sent_to, subject, body, content='messages', content_rowid='id', tokenize='unicode61' ); CREATE TRIGGER IF NOT EXISTS messages_ai AFTER INSERT ON messages BEGIN INSERT INTO messages_fts(rowid, sent_from, sent_to, subject, body) VALUES (new.id, new.sent_from, new.sent_to, new.subject, new.body); END; CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN INSERT INTO messages_fts(messages_fts, rowid, sent_from, sent_to, subject, body) VALUES ('delete', old.id, old.sent_from, old.sent_to, old.subject, old.body); END; CREATE TRIGGER IF NOT EXISTS messages_au AFTER UPDATE ON messages BEGIN INSERT INTO messages_fts(messages_fts, rowid, sent_from, sent_to, subject, body) VALUES ('delete', old.id, old.sent_from, old.sent_to, old.subject, old.body); INSERT INTO messages_fts(rowid, sent_from, sent_to, subject, body) VALUES (new.id, new.sent_from, new.sent_to, new.subject, new.body); END; """ def _term_list(query): """Nicht-alphanumerische Zeichen als Trenner; Lowercase.""" return [t for t in re.split(r"[^0-9A-Za-z_]+", query.lower()) if t] def _decode_hdr(raw): from email.header import decode_header if not raw: return "" parts = decode_header(str(raw)) out = [] for data, charset in parts: if isinstance(data, bytes): for cs in (charset, "utf-8", "latin-1"): if not cs: continue try: out.append(data.decode(cs, errors="replace")) break except (LookupError, TypeError): continue else: out.append(data.decode("latin-1", errors="replace")) else: out.append(str(data)) return " ".join(out) def _extract_textparts(msg): """Text-Bodies sammeln: text/plain bevorzugt, sonst text/html (ohne Tags).""" plains, htmls = [], [] if msg.is_multipart(): for part in msg.walk(): ct = part.get_content_type() payload = part.get_payload(decode=True) if not payload: continue try: text = payload.decode(part.get_content_charset() or "utf-8", errors="replace") except (LookupError, TypeError): text = payload.decode("latin-1", errors="replace") if ct == "text/plain": plains.append(text) elif ct == "text/html": htmls.append(text) else: payload = msg.get_payload(decode=True) if payload: try: plains.append(payload.decode(msg.get_content_charset() or "utf-8", errors="replace")) except (LookupError, TypeError): plains.append(payload.decode("latin-1", errors="replace")) if plains: return "\n".join(plains) if htmls: out = [] for h in htmls: h = re.sub(r"", " ", h, flags=re.S | re.I) h = re.sub(r"", " ", h, flags=re.S | re.I) h = re.sub(r"<[^>]+>", " ", h) out.append(htmlmod.unescape(h)) return "\n".join(out) return "" def _parse_file(path, root): """Mail-Datei einlesen → (account, folder, key, date_ts, sender, to, subject, body). Pfadform unter root: //cur/ """ rel = os.path.relpath(path, root) parts = rel.split(os.sep) # erwartet: [account, folder, "cur", key] — aber Ordner koennen verschachtelt sein key = parts[-1] folder = os.sep.join(parts[1:-2]) if len(parts) > 3 else parts[-3] account = parts[0] try: with open(path, "rb") as f: raw = f.read() except OSError: return None try: msg = BytesParser(policy=policy.default).parsebytes(raw) except Exception: return None date_str = str(msg.get("Date", "")) date_ts = 0 if date_str: try: dt = parsedate_to_datetime(date_str) if dt: date_ts = int(dt.timestamp()) except Exception: pass sender = _decode_hdr(msg.get("From", "")) to = _decode_hdr(msg.get("To", "")) subject = _decode_hdr(msg.get("Subject", "")) body = _extract_textparts(msg) return account, folder, key, date_ts, sender, to, subject, body def index_db_path(user): return os.path.join(INDEX_DIR, f"{user}.sqlite") def _ready_path(user): """Marker: Erst-Build abgeschlossen → Suche darf den Index nutzen.""" return os.path.join(INDEX_DIR, f"{user}.ready") def connect(user, create=True): os.makedirs(INDEX_DIR, exist_ok=True) db = index_db_path(user) con = sqlite3.connect(db, timeout=30) con.execute("PRAGMA journal_mode=WAL") con.execute("PRAGMA synchronous=NORMAL") if create: con.executescript(_SCHEMA) return con def update_user(user, root, log=print): """Inkrementeller Index-Update fuer einen User-Maildir-Root. Gibt (added, updated, removed) zurueck. """ if not os.path.isdir(root): log(f" [{user}] Root fehlt: {root}") return 0, 0, 0 con = connect(user) try: seen = set() added = updated = removed = 0 # cur/ + new/-Dateien aller Ordner einsammeln (followlinks: erlaubt Account-Symlinks wie nova->.com). # new/ = ungelesene Mails (mbsync legt \Unseen dort ab); ohne new/ fehlten sie im Index. for dirpath, dirnames, filenames in os.walk(root, followlinks=True): if os.path.basename(dirpath) not in ("cur", "new"): continue for fn in filenames: path = os.path.join(dirpath, fn) try: st = os.stat(path) except OSError: continue mtime, size = st.st_mtime, st.st_size seen.add(path) row = con.execute( "SELECT id FROM messages WHERE path=?", (path,) ).fetchone() if row: cur = con.execute( "SELECT mtime, size FROM messages WHERE id=?", (row[0],) ).fetchone() if cur and abs(cur[0] - mtime) < 0.001 and cur[1] == size: continue # unveraendert updated += 1 elif size == 0: continue # leere Datei beim Sync noch nicht fertig else: added += 1 parsed = _parse_file(path, root) if not parsed: con.execute("DELETE FROM messages WHERE path=?", (path,)) continue account, folder, key, date_ts, sender, to, subject, body = parsed con.execute( """INSERT INTO messages (path, account, folder, key, mtime, size, date_ts, sent_from, sent_to, subject, body) VALUES (?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(path) DO UPDATE SET account=excluded.account, folder=excluded.folder, key=excluded.key, mtime=excluded.mtime, size=excluded.size, date_ts=excluded.date_ts, sent_from=excluded.sent_from, sent_to=excluded.sent_to, subject=excluded.subject, body=excluded.body""", (path, account, folder, key, mtime, size, date_ts, sender, to, subject, body), ) # in DB, aber Datei weg → loeschen for (path,) in con.execute("SELECT path FROM messages").fetchall(): if path not in seen: con.execute("DELETE FROM messages WHERE path=?", (path,)) removed += 1 con.commit() return added, updated, removed finally: con.close() def _fts_query(query): """FTS5-MATCH-String: alle Terme als Praefix-OR innerhalb eines Terms (ADDED-Optimierung: mehrere Woerter muessen ALLE vorkommen → AND).""" terms = _term_list(query) if not terms: return None # jedes Wort als Prefix-Term: "term"* — escapen von Anführungszeichen parts = [] for t in terms: t = t.replace('"', "") parts.append(f'"{t}"*') return " AND ".join(parts) def search(user, query, account="", folder="", limit=20, log=print): """Volltextsuche im Index. Liefert Liste von Dicts {account, folder, key, date, sender, to, subject} — gleiches Format wie der alte Vollscan, damit search_mail unveraendert weitermacht. """ db = index_db_path(user) if not os.path.exists(db) or not os.path.exists(_ready_path(user)): return None # Index noch nicht (vollstaendig) gebaut → Fallback im Server con = connect(user, create=False) try: term_like = f"%{query.lower()}%" base = "SELECT account, folder, key, date_ts, sent_from, sent_to, subject FROM messages" where_clauses = [] params = [] # E-Mail-Adressen: FTS5 zerlegt sie in Einzelwoerter mit zu striktem # AND → direkt ueber den LIKE-Pfad (exakt auf sender/to/subject). has_email = "@" in query if not has_email: fts = _fts_query(query) if fts: where_clauses.append("id IN (SELECT rowid FROM messages_fts WHERE messages_fts MATCH ?)") params.append(fts) if account: where_clauses.append("account=?") params.append(account) if folder: where_clauses.append("folder LIKE ?") params.append(f"%{folder}%") out = [] if where_clauses: where_sql = " WHERE " + " AND ".join(where_clauses) rows = con.execute( f"{base}{where_sql} ORDER BY date_ts DESC, id DESC LIMIT ?", params + [limit], ).fetchall() for account_, folder_, key_, date_ts, sender, to, subject in rows: out.append({ "account": account_, "folder": folder_, "key": key_, "date": _fmt_date(date_ts), "sender": sender, "to": to, "subject": subject, }) # Fallback: LIKE-Suche (deckt Substrings + E-Mail-Adressen ab) if not out: cols = ["sent_from LIKE ?", "sent_to LIKE ?", "subject LIKE ?"] all_p = [term_like] * len(cols) extra_where = [] if account: extra_where.append("account=?") all_p.append(account) if folder: extra_where.append("folder LIKE ?") all_p.append(f"%{folder}%") sql = f"{base} WHERE ({' OR '.join(cols)})" if extra_where: sql += " AND " + " AND ".join(extra_where) sql += " ORDER BY date_ts DESC, id DESC LIMIT ?" all_p.append(limit) rows2 = con.execute(sql, all_p).fetchall() for account_, folder_, key_, date_ts, sender, to, subject in rows2: out.append({ "account": account_, "folder": folder_, "key": key_, "date": _fmt_date(date_ts), "sender": sender, "to": to, "subject": subject, }) return out finally: con.close() def _fmt_date(ts): if not ts: return "" return time.strftime("%a, %d %b %Y %H:%M:%S %z", time.localtime(ts)) if __name__ == "__main__": # CLI: python3 mailindex.py --update|--search "" [limit] import argparse ap = argparse.ArgumentParser() ap.add_argument("user") ap.add_argument("--update", action="store_true", help="inkrementeller Update") ap.add_argument("--search", nargs="?", default=None) ap.add_argument("--limit", type=int, default=10) args = ap.parse_args() cfg = _lc() root = cfg["mail_roots"].get(args.user) if args.update: if not root: print(f"kein root fuer {args.user}") sys.exit(1) a, u, r = update_user(args.user, root) print(f"[{args.user}] added={a} updated={u} removed={r}") elif args.search: res = search(args.user, args.search, limit=args.limit) if res is None: print("kein index") elif not res: print("keine treffer") else: for r in res: print(f"[{r['date']}] {r['sender']} -> {r['to']}\n {r['subject']}\n {r['account']}/{r['folder']}/{r['key']}") else: ap.print_help()