fix(calendar): TZID Europe/Berlin for timed events/tasks, resolve real resource href on delete/update, fix _report_tasks XML; sync deployed servers (owamail, openwearables, ghostfolio), docs and tests
This commit is contained in:
1 parent
7c01e7be53
commit
75185ef2ba
41 files changed
+8834
-324
No files matched your search
@@ -0,0 +1,377 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Maildir-Volltextindex (SQLite FTS5) fuer den MCP-Mail-Server.
|
||||
|
||||
Hintergrund: search_mail scannte bei jeder Anfrage das komplette Maildir
|
||||
(Roh-Byte-Vorfilter ueber alle Dateien). Bei Katis Gmail-Archiv (16 GB,
|
||||
~30k Mails) dauerte das laenger als das Gateway-Request-Timeout → "Request
|
||||
timed out". Dieser Index macht Suchen zu FTS5-SQL-Queries (Millisekunden).
|
||||
|
||||
Design:
|
||||
- Eine SQLite-DB pro User unter /var/lib/mail-index/<user>.sqlite
|
||||
- Tabelle messages (Stammdaten + body) + virtuelle Tabelle messages_fts
|
||||
(FTS5, content='messages', mit Trigger-Sync via externem Content)
|
||||
- Inkrementell: Datei-Pfad + mtime + size in messages; beim Update nur
|
||||
neue/geaenderte Dateien neu einlesen, entfernte loeschen
|
||||
- Such-Strategie: FTS5-MATCH mit Prefix-Termen; Fallback auf LIKE ueber
|
||||
sender/to/subject (deckt Substrings und E-Mail-Adressen ab), zuletzt
|
||||
LIKE ueber body
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
import html as htmlmod
|
||||
from email import policy
|
||||
from email.parser import BytesParser
|
||||
from email.utils import parsedate_to_datetime
|
||||
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, BASE_DIR)
|
||||
from common import load_config as _lc # noqa: E402
|
||||
|
||||
INDEX_DIR = "/var/lib/mail-index"
|
||||
_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id INTEGER PRIMARY KEY,
|
||||
path TEXT UNIQUE NOT NULL,
|
||||
account TEXT NOT NULL,
|
||||
folder TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
mtime REAL NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
date_ts INTEGER DEFAULT 0,
|
||||
sent_from TEXT DEFAULT '',
|
||||
sent_to TEXT DEFAULT '',
|
||||
subject TEXT DEFAULT '',
|
||||
body TEXT DEFAULT ''
|
||||
);
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
|
||||
sent_from, sent_to, subject, body,
|
||||
content='messages', content_rowid='id',
|
||||
tokenize='unicode61'
|
||||
);
|
||||
CREATE TRIGGER IF NOT EXISTS messages_ai AFTER INSERT ON messages BEGIN
|
||||
INSERT INTO messages_fts(rowid, sent_from, sent_to, subject, body)
|
||||
VALUES (new.id, new.sent_from, new.sent_to, new.subject, new.body);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN
|
||||
INSERT INTO messages_fts(messages_fts, rowid, sent_from, sent_to, subject, body)
|
||||
VALUES ('delete', old.id, old.sent_from, old.sent_to, old.subject, old.body);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS messages_au AFTER UPDATE ON messages BEGIN
|
||||
INSERT INTO messages_fts(messages_fts, rowid, sent_from, sent_to, subject, body)
|
||||
VALUES ('delete', old.id, old.sent_from, old.sent_to, old.subject, old.body);
|
||||
INSERT INTO messages_fts(rowid, sent_from, sent_to, subject, body)
|
||||
VALUES (new.id, new.sent_from, new.sent_to, new.subject, new.body);
|
||||
END;
|
||||
"""
|
||||
|
||||
|
||||
def _term_list(query):
|
||||
"""Nicht-alphanumerische Zeichen als Trenner; Lowercase."""
|
||||
return [t for t in re.split(r"[^0-9A-Za-z_]+", query.lower()) if t]
|
||||
|
||||
|
||||
def _decode_hdr(raw):
|
||||
from email.header import decode_header
|
||||
if not raw:
|
||||
return ""
|
||||
parts = decode_header(str(raw))
|
||||
out = []
|
||||
for data, charset in parts:
|
||||
if isinstance(data, bytes):
|
||||
for cs in (charset, "utf-8", "latin-1"):
|
||||
if not cs:
|
||||
continue
|
||||
try:
|
||||
out.append(data.decode(cs, errors="replace"))
|
||||
break
|
||||
except (LookupError, TypeError):
|
||||
continue
|
||||
else:
|
||||
out.append(data.decode("latin-1", errors="replace"))
|
||||
else:
|
||||
out.append(str(data))
|
||||
return " ".join(out)
|
||||
|
||||
|
||||
def _extract_textparts(msg):
|
||||
"""Text-Bodies sammeln: text/plain bevorzugt, sonst text/html (ohne Tags)."""
|
||||
plains, htmls = [], []
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_type()
|
||||
payload = part.get_payload(decode=True)
|
||||
if not payload:
|
||||
continue
|
||||
try:
|
||||
text = payload.decode(part.get_content_charset() or "utf-8", errors="replace")
|
||||
except (LookupError, TypeError):
|
||||
text = payload.decode("latin-1", errors="replace")
|
||||
if ct == "text/plain":
|
||||
plains.append(text)
|
||||
elif ct == "text/html":
|
||||
htmls.append(text)
|
||||
else:
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload:
|
||||
try:
|
||||
plains.append(payload.decode(msg.get_content_charset() or "utf-8", errors="replace"))
|
||||
except (LookupError, TypeError):
|
||||
plains.append(payload.decode("latin-1", errors="replace"))
|
||||
if plains:
|
||||
return "\n".join(plains)
|
||||
if htmls:
|
||||
out = []
|
||||
for h in htmls:
|
||||
h = re.sub(r"<style.*?</style>", " ", h, flags=re.S | re.I)
|
||||
h = re.sub(r"<script.*?</script>", " ", h, flags=re.S | re.I)
|
||||
h = re.sub(r"<[^>]+>", " ", h)
|
||||
out.append(htmlmod.unescape(h))
|
||||
return "\n".join(out)
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_file(path, root):
|
||||
"""Mail-Datei einlesen → (account, folder, key, date_ts, sender, to, subject, body).
|
||||
|
||||
Pfadform unter root: <account>/<folder>/cur/<key>
|
||||
"""
|
||||
rel = os.path.relpath(path, root)
|
||||
parts = rel.split(os.sep)
|
||||
# erwartet: [account, folder, "cur", key] — aber Ordner koennen verschachtelt sein
|
||||
key = parts[-1]
|
||||
folder = os.sep.join(parts[1:-2]) if len(parts) > 3 else parts[-3]
|
||||
account = parts[0]
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
raw = f.read()
|
||||
except OSError:
|
||||
return None
|
||||
try:
|
||||
msg = BytesParser(policy=policy.default).parsebytes(raw)
|
||||
except Exception:
|
||||
return None
|
||||
date_str = str(msg.get("Date", ""))
|
||||
date_ts = 0
|
||||
if date_str:
|
||||
try:
|
||||
dt = parsedate_to_datetime(date_str)
|
||||
if dt:
|
||||
date_ts = int(dt.timestamp())
|
||||
except Exception:
|
||||
pass
|
||||
sender = _decode_hdr(msg.get("From", ""))
|
||||
to = _decode_hdr(msg.get("To", ""))
|
||||
subject = _decode_hdr(msg.get("Subject", ""))
|
||||
body = _extract_textparts(msg)
|
||||
return account, folder, key, date_ts, sender, to, subject, body
|
||||
|
||||
|
||||
def index_db_path(user):
|
||||
return os.path.join(INDEX_DIR, f"{user}.sqlite")
|
||||
|
||||
|
||||
def _ready_path(user):
|
||||
"""Marker: Erst-Build abgeschlossen → Suche darf den Index nutzen."""
|
||||
return os.path.join(INDEX_DIR, f"{user}.ready")
|
||||
|
||||
|
||||
def connect(user, create=True):
|
||||
os.makedirs(INDEX_DIR, exist_ok=True)
|
||||
db = index_db_path(user)
|
||||
con = sqlite3.connect(db, timeout=30)
|
||||
con.execute("PRAGMA journal_mode=WAL")
|
||||
con.execute("PRAGMA synchronous=NORMAL")
|
||||
if create:
|
||||
con.executescript(_SCHEMA)
|
||||
return con
|
||||
|
||||
|
||||
def update_user(user, root, log=print):
|
||||
"""Inkrementeller Index-Update fuer einen User-Maildir-Root.
|
||||
|
||||
Gibt (added, updated, removed) zurueck.
|
||||
"""
|
||||
if not os.path.isdir(root):
|
||||
log(f" [{user}] Root fehlt: {root}")
|
||||
return 0, 0, 0
|
||||
con = connect(user)
|
||||
try:
|
||||
seen = set()
|
||||
added = updated = removed = 0
|
||||
# cur/ + new/-Dateien aller Ordner einsammeln (followlinks: erlaubt Account-Symlinks wie nova->.com).
|
||||
# new/ = ungelesene Mails (mbsync legt \Unseen dort ab); ohne new/ fehlten sie im Index.
|
||||
for dirpath, dirnames, filenames in os.walk(root, followlinks=True):
|
||||
if os.path.basename(dirpath) not in ("cur", "new"):
|
||||
continue
|
||||
for fn in filenames:
|
||||
path = os.path.join(dirpath, fn)
|
||||
try:
|
||||
st = os.stat(path)
|
||||
except OSError:
|
||||
continue
|
||||
mtime, size = st.st_mtime, st.st_size
|
||||
seen.add(path)
|
||||
row = con.execute(
|
||||
"SELECT id FROM messages WHERE path=?", (path,)
|
||||
).fetchone()
|
||||
if row:
|
||||
cur = con.execute(
|
||||
"SELECT mtime, size FROM messages WHERE id=?", (row[0],)
|
||||
).fetchone()
|
||||
if cur and abs(cur[0] - mtime) < 0.001 and cur[1] == size:
|
||||
continue # unveraendert
|
||||
updated += 1
|
||||
elif size == 0:
|
||||
continue # leere Datei beim Sync noch nicht fertig
|
||||
else:
|
||||
added += 1
|
||||
parsed = _parse_file(path, root)
|
||||
if not parsed:
|
||||
con.execute("DELETE FROM messages WHERE path=?", (path,))
|
||||
continue
|
||||
account, folder, key, date_ts, sender, to, subject, body = parsed
|
||||
con.execute(
|
||||
"""INSERT INTO messages
|
||||
(path, account, folder, key, mtime, size, date_ts,
|
||||
sent_from, sent_to, subject, body)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(path) DO UPDATE SET
|
||||
account=excluded.account, folder=excluded.folder,
|
||||
key=excluded.key, mtime=excluded.mtime, size=excluded.size,
|
||||
date_ts=excluded.date_ts, sent_from=excluded.sent_from,
|
||||
sent_to=excluded.sent_to, subject=excluded.subject,
|
||||
body=excluded.body""",
|
||||
(path, account, folder, key, mtime, size, date_ts,
|
||||
sender, to, subject, body),
|
||||
)
|
||||
# in DB, aber Datei weg → loeschen
|
||||
for (path,) in con.execute("SELECT path FROM messages").fetchall():
|
||||
if path not in seen:
|
||||
con.execute("DELETE FROM messages WHERE path=?", (path,))
|
||||
removed += 1
|
||||
con.commit()
|
||||
return added, updated, removed
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def _fts_query(query):
|
||||
"""FTS5-MATCH-String: alle Terme als Praefix-OR innerhalb eines Terms
|
||||
(ADDED-Optimierung: mehrere Woerter muessen ALLE vorkommen → AND)."""
|
||||
terms = _term_list(query)
|
||||
if not terms:
|
||||
return None
|
||||
# jedes Wort als Prefix-Term: "term"* — escapen von Anführungszeichen
|
||||
parts = []
|
||||
for t in terms:
|
||||
t = t.replace('"', "")
|
||||
parts.append(f'"{t}"*')
|
||||
return " AND ".join(parts)
|
||||
|
||||
|
||||
def search(user, query, account="", folder="", limit=20, log=print):
|
||||
"""Volltextsuche im Index. Liefert Liste von Dicts
|
||||
{account, folder, key, date, sender, to, subject} — gleiches Format
|
||||
wie der alte Vollscan, damit search_mail unveraendert weitermacht.
|
||||
"""
|
||||
db = index_db_path(user)
|
||||
if not os.path.exists(db) or not os.path.exists(_ready_path(user)):
|
||||
return None # Index noch nicht (vollstaendig) gebaut → Fallback im Server
|
||||
con = connect(user, create=False)
|
||||
try:
|
||||
term_like = f"%{query.lower()}%"
|
||||
base = "SELECT account, folder, key, date_ts, sent_from, sent_to, subject FROM messages"
|
||||
where_clauses = []
|
||||
params = []
|
||||
# E-Mail-Adressen: FTS5 zerlegt sie in Einzelwoerter mit zu striktem
|
||||
# AND → direkt ueber den LIKE-Pfad (exakt auf sender/to/subject).
|
||||
has_email = "@" in query
|
||||
if not has_email:
|
||||
fts = _fts_query(query)
|
||||
if fts:
|
||||
where_clauses.append("id IN (SELECT rowid FROM messages_fts WHERE messages_fts MATCH ?)")
|
||||
params.append(fts)
|
||||
if account:
|
||||
where_clauses.append("account=?")
|
||||
params.append(account)
|
||||
if folder:
|
||||
where_clauses.append("folder LIKE ?")
|
||||
params.append(f"%{folder}%")
|
||||
out = []
|
||||
if where_clauses:
|
||||
where_sql = " WHERE " + " AND ".join(where_clauses)
|
||||
rows = con.execute(
|
||||
f"{base}{where_sql} ORDER BY date_ts DESC, id DESC LIMIT ?",
|
||||
params + [limit],
|
||||
).fetchall()
|
||||
for account_, folder_, key_, date_ts, sender, to, subject in rows:
|
||||
out.append({
|
||||
"account": account_, "folder": folder_, "key": key_,
|
||||
"date": _fmt_date(date_ts), "sender": sender, "to": to,
|
||||
"subject": subject,
|
||||
})
|
||||
# Fallback: LIKE-Suche (deckt Substrings + E-Mail-Adressen ab)
|
||||
if not out:
|
||||
cols = ["sent_from LIKE ?", "sent_to LIKE ?", "subject LIKE ?"]
|
||||
all_p = [term_like] * len(cols)
|
||||
extra_where = []
|
||||
if account:
|
||||
extra_where.append("account=?")
|
||||
all_p.append(account)
|
||||
if folder:
|
||||
extra_where.append("folder LIKE ?")
|
||||
all_p.append(f"%{folder}%")
|
||||
sql = f"{base} WHERE ({' OR '.join(cols)})"
|
||||
if extra_where:
|
||||
sql += " AND " + " AND ".join(extra_where)
|
||||
sql += " ORDER BY date_ts DESC, id DESC LIMIT ?"
|
||||
all_p.append(limit)
|
||||
rows2 = con.execute(sql, all_p).fetchall()
|
||||
for account_, folder_, key_, date_ts, sender, to, subject in rows2:
|
||||
out.append({
|
||||
"account": account_, "folder": folder_, "key": key_,
|
||||
"date": _fmt_date(date_ts), "sender": sender, "to": to,
|
||||
"subject": subject,
|
||||
})
|
||||
return out
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def _fmt_date(ts):
|
||||
if not ts:
|
||||
return ""
|
||||
return time.strftime("%a, %d %b %Y %H:%M:%S %z", time.localtime(ts))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# CLI: python3 mailindex.py <user> --update|--search "<query>" [limit]
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("user")
|
||||
ap.add_argument("--update", action="store_true", help="inkrementeller Update")
|
||||
ap.add_argument("--search", nargs="?", default=None)
|
||||
ap.add_argument("--limit", type=int, default=10)
|
||||
args = ap.parse_args()
|
||||
cfg = _lc()
|
||||
root = cfg["mail_roots"].get(args.user)
|
||||
if args.update:
|
||||
if not root:
|
||||
print(f"kein root fuer {args.user}")
|
||||
sys.exit(1)
|
||||
a, u, r = update_user(args.user, root)
|
||||
print(f"[{args.user}] added={a} updated={u} removed={r}")
|
||||
elif args.search:
|
||||
res = search(args.user, args.search, limit=args.limit)
|
||||
if res is None:
|
||||
print("kein index")
|
||||
elif not res:
|
||||
print("keine treffer")
|
||||
else:
|
||||
for r in res:
|
||||
print(f"[{r['date']}] {r['sender']} -> {r['to']}\n {r['subject']}\n {r['account']}/{r['folder']}/{r['key']}")
|
||||
else:
|
||||
ap.print_help()
|
||||
Reference in new issue
Block a user