From 3b3aff430c0c1213c75e1cac0df3440ac1f78099 Mon Sep 17 00:00:00 2001 From: Lunixizm0 <216941080+Lunixizm0@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:13:44 +0300 Subject: [PATCH 1/4] feat(cert_tr): Implement Proton Mail integration for CERT-TR advisory fetching --- .env.example | 20 ++ release.sh | 2 +- src/advisory_rss/cache/models.py | 5 + src/advisory_rss/cache/store.py | 36 ++- src/advisory_rss/cert_tr/__init__.py | 1 + src/advisory_rss/cert_tr/imap.py | 208 +++++++++++++ src/advisory_rss/cert_tr/parser.py | 447 +++++++++++++++++++++++++++ src/advisory_rss/cert_tr/source.py | 111 +++++++ src/advisory_rss/cli.py | 246 ++++++++++----- src/advisory_rss/config/constants.py | 11 + src/advisory_rss/config/settings.py | 183 +++++++++++ src/advisory_rss/rss/builder.py | 27 +- src/advisory_rss/server/app.py | 236 +++++++++++--- tests/unit/test_cert_tr_imap.py | 60 ++++ tests/unit/test_cert_tr_parser.py | 53 ++++ tests/unit/test_cert_tr_rss.py | 47 +++ tests/unit/test_cert_tr_settings.py | 52 ++++ 17 files changed, 1622 insertions(+), 123 deletions(-) create mode 100644 src/advisory_rss/cert_tr/__init__.py create mode 100644 src/advisory_rss/cert_tr/imap.py create mode 100644 src/advisory_rss/cert_tr/parser.py create mode 100644 src/advisory_rss/cert_tr/source.py create mode 100644 tests/unit/test_cert_tr_imap.py create mode 100644 tests/unit/test_cert_tr_parser.py create mode 100644 tests/unit/test_cert_tr_rss.py create mode 100644 tests/unit/test_cert_tr_settings.py diff --git a/.env.example b/.env.example index 3957d29..4753641 100644 --- a/.env.example +++ b/.env.example @@ -50,3 +50,23 @@ CACHE_PATH=cache/advisories.db # Hard caps to bound pagination (avoid silently hitting rate limits without warning) # MAX_REPOS=1000 # MAX_PAGES_PER_REPO=100 + +# CERT-TR / Siber Güvenlik Başkanlığı via Proton Mail Bridge (localhost IMAP) +# When false (default) only GitHub source is active. RSS is mixed (GitHub + CERT-TR) when enabled. +# ENABLE_CERT_TR=false +# For single account: +# PROTON_BRIDGE_EMAIL=mail@proton.me +# PROTON_BRIDGE_PASSWORD=your_bridge_generated_password # Bridge -> Settings -> Mailbox details (NOT account password) +# PROTON_IMAP_FOLDER=INBOX # Which folder/label to scan (INBOX, INBOX.CERT-TR, etc.) +# For multiple accounts (each mail + inbox separately configurable, never marks as read): +# PROTON_BRIDGE_EMAILS=mail@proton.me,second@proton.me +# PROTON_BRIDGE_PASSWORDS=bridge_pass1,bridge_pass2 # index-aligned with EMAILS +# PROTON_IMAP_FOLDERS=INBOX,INBOX.CERT-TR # index-aligned; missing -> defaults to PROTON_IMAP_FOLDER +# Bridge connection (usually don't change): +# PROTON_BRIDGE_HOST=127.0.0.1 +# PROTON_IMAP_PORT=1143 +# PROTON_IMAP_SECURITY=STARTTLS # STARTTLS | SSL | NONE (Bridge uses STARTTLS, self-signed) +# Filtering & limits: +# CERT_TR_SENDER_ALLOWLIST=siberguvenlik.gov.tr +# CERT_TR_MAX_MAILS=200 +# CERT_TR_SEARCH_DAYS= # empty = all time, or e.g. 30 for last 30 days diff --git a/release.sh b/release.sh index 1d7efb8..988f532 100755 --- a/release.sh +++ b/release.sh @@ -22,7 +22,7 @@ echo "Tag oluşturuluyor..." git tag -f "${TAG}" echo "Push ediliyor..." -git push origin master "${TAG}" +git push origin main "${TAG}" echo "GitHub Release oluşturuluyor..." gh release delete "${TAG}" --yes --cleanup-tag 2>/dev/null || true diff --git a/src/advisory_rss/cache/models.py b/src/advisory_rss/cache/models.py index eb544b4..d8f2a86 100644 --- a/src/advisory_rss/cache/models.py +++ b/src/advisory_rss/cache/models.py @@ -32,6 +32,9 @@ class NormalizedAdvisory: publisher_login: str | None = None identifiers: list[dict[str, str]] = field(default_factory=list) raw: dict[str, Any] = field(default_factory=dict, repr=False) + # Multi-source extension + source: str = "github" + extra_cves: list[str] = field(default_factory=list) def sort_key(self) -> datetime: # Use updated_at > published_at > created_at > distant past @@ -92,6 +95,8 @@ def _parse(v: str | None) -> datetime | None: publisher_login=data.get("publisher_login"), identifiers=list(data.get("identifiers") or []), raw=data.get("raw") or {}, + source=data.get("source") or "github", + extra_cves=list(data.get("extra_cves") or []), ) return adv diff --git a/src/advisory_rss/cache/store.py b/src/advisory_rss/cache/store.py index 4862dda..20eb983 100644 --- a/src/advisory_rss/cache/store.py +++ b/src/advisory_rss/cache/store.py @@ -42,7 +42,8 @@ def _connect(db_path: Path) -> sqlite3.Connection: updated_at TEXT, state TEXT, html_url TEXT, - author_login TEXT + author_login TEXT, + source TEXT DEFAULT 'github' ); CREATE TABLE IF NOT EXISTS meta( key TEXT PRIMARY KEY, @@ -66,6 +67,31 @@ def _init_schema(self) -> None: logger.debug("init schema db=%s", self.db_path) try: self._conn.executescript(SCHEMA) + # Lightweight migrations for existing DBs (idempotent) + try: + cols = {r[1] for r in self._conn.execute("PRAGMA table_info(advisories)").fetchall()} + if "source" not in cols: + self._conn.execute("ALTER TABLE advisories ADD COLUMN source TEXT DEFAULT 'github'") + logger.info("Migrated advisories: added source column") + # Backfill normalized_json missing source field (always, idempotent) + try: + cur = self._conn.execute("SELECT ghsa_id, normalized_json FROM advisories") + for ghsa_id, nj in cur.fetchall(): + try: + d = json.loads(nj) + if "source" not in d: + d["source"] = "github" + new_nj = json.dumps(d, ensure_ascii=False) + self._conn.execute( + "UPDATE advisories SET normalized_json=?, source=? WHERE ghsa_id=?", + (new_nj, "github", ghsa_id), + ) + except (json.JSONDecodeError, ValueError, TypeError, sqlite3.Error) as ee: + logger.debug("migration backfill skipped %s: %s", ghsa_id, ee) + except sqlite3.Error as ee: + logger.debug("backfill query failed: %s", ee) + except sqlite3.Error as me: + logger.warning("Migration check failed: %s", me) logger.debug("cache schema ready db=%s", self.db_path) except sqlite3.Error as e: logger.error("Cache schema init failed: %s", e, exc_info=True) @@ -88,15 +114,16 @@ def upsert_advisories(self, advisories: list[NormalizedAdvisory]) -> int: updated_iso = adv.updated_at.isoformat() if adv.updated_at else None cur.execute( """ - INSERT INTO advisories(ghsa_id, raw_json, normalized_json, updated_at, state, html_url, author_login) - VALUES (?, ?, ?, ?, ?, ?, ?) + INSERT INTO advisories(ghsa_id, raw_json, normalized_json, updated_at, state, html_url, author_login, source) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(ghsa_id) DO UPDATE SET raw_json=excluded.raw_json, normalized_json=excluded.normalized_json, updated_at=excluded.updated_at, state=excluded.state, html_url=excluded.html_url, - author_login=excluded.author_login + author_login=excluded.author_login, + source=excluded.source """, ( adv.ghsa_id, @@ -106,6 +133,7 @@ def upsert_advisories(self, advisories: list[NormalizedAdvisory]) -> int: adv.state, adv.html_url, adv.author_login, + getattr(adv, "source", "github") or "github", ), ) count += 1 diff --git a/src/advisory_rss/cert_tr/__init__.py b/src/advisory_rss/cert_tr/__init__.py new file mode 100644 index 0000000..858031d --- /dev/null +++ b/src/advisory_rss/cert_tr/__init__.py @@ -0,0 +1 @@ +"""CERT-TR / SGB Proton Mail source package.""" diff --git a/src/advisory_rss/cert_tr/imap.py b/src/advisory_rss/cert_tr/imap.py new file mode 100644 index 0000000..7f9916a --- /dev/null +++ b/src/advisory_rss/cert_tr/imap.py @@ -0,0 +1,208 @@ +"""Proton Bridge IMAP fetcher - localhost only, never marks read.""" +# ruff: noqa: BLE001, DTZ003, S110, RUF059 + +from __future__ import annotations + +import imaplib +import logging +import ssl +import time +from datetime import datetime, timedelta + +logger = logging.getLogger(__name__) + +# Never mark read: we always use BODY.PEEK and SELECT readonly=True + + +def _ssl_context() -> ssl.SSLContext: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx + + +def connect_imap(account: dict[str, str]) -> imaplib.IMAP4: + host = account.get("host", "127.0.0.1") + port = int(account.get("port", 1143)) + security = (account.get("security") or "STARTTLS").upper() + email = account.get("email", "") + password = account.get("password", "") + + # Loopback guard - only allow loopback + if host not in ("127.0.0.1", "::1", "localhost"): + raise ValueError(f"Proton IMAP host must be loopback, got {host!r}") + + ctx = _ssl_context() + if security == "SSL": + mail = imaplib.IMAP4_SSL(host, port, ssl_context=ctx) + elif security == "STARTTLS": + mail = imaplib.IMAP4(host, port) + try: + mail.starttls(ssl_context=ctx) + except Exception as e: + logger.warning("STARTTLS failed for %s:%s - trying without TLS: %s", host, port, e) + # already connected without TLS; still try login + else: # NONE + mail = imaplib.IMAP4(host, port) + + # Login - Bridge password is not Proton account password + try: + mail.login(email, password) + except imaplib.IMAP4.error as e: + logger.error("IMAP login failed for %s: %s", email, e) + raise + logger.debug("IMAP connected %s folder will be selected later", email) + return mail + + +def _build_search_criteria(allowlist: list[str], search_days: int | None) -> list[str]: + parts: list[str] = [] + # FROM filter - build OR chain if multiple + if allowlist: + # Use first allowlist entry for server-side filter; rest filtered client-side to keep simple + # Or build OR for 2 entries max? We'll use first dominant domain + domain = allowlist[0] + # IMAP FROM search is substring match; domain is enough + parts.append(f'FROM "{domain}"') + if search_days is not None and search_days >= 0: + # SINCE date is inclusive + since_date = (datetime.utcnow() - timedelta(days=search_days)).strftime("%d-%b-%Y") + parts.append(f'SINCE "{since_date}"') + if not parts: + parts.append("ALL") + return parts + + +def fetch_raw_emails( + account: dict[str, str], + allowlist: list[str], + max_mails: int = 200, + search_days: int | None = None, +) -> list[bytes]: + """Fetch raw RFC822 bytes for one Proton account, never marking as read. + + Returns list of raw bytes, newest first (reverse UID). + Handles reconnect every 25 to avoid Bridge drops. + """ + folder = account.get("folder", "INBOX") or "INBOX" + email = account.get("email", "unknown") + raw_list: list[bytes] = [] + + mail = None + try: + mail = connect_imap(account) + # Select readonly to guarantee no \Seen flag can be set even on bug + typ: str = "" + data: list[bytes] = [] + try: + typ, data = mail.select(folder, readonly=True) # type: ignore[assignment] + except imaplib.IMAP4.error as e: # noqa: BLE001 + logger.warning("IMAP select folder %r failed for %s: %s", folder, email, e) + # try INBOX fallback + if folder.upper() != "INBOX": + try: + typ2, _data2 = mail.select("INBOX", readonly=True) # type: ignore[assignment] + if typ2 == "OK": + logger.info("Fallback to INBOX for %s", email) + folder = "INBOX" + typ, data = typ2, _data2 # type: ignore[assignment] + else: + return [] + except Exception: # noqa: BLE001 + return [] + else: + return [] + if typ != "OK": + logger.warning("IMAP select %r returned %s for %s: %s", folder, typ, email, data) + return [] + + # Build search + criteria = _build_search_criteria(allowlist, search_days) + # IMAP search: join with space (AND) + search_query = " ".join(criteria) if len(criteria) == 1 else "(" + " ".join(criteria) + ")" + # Actually need to handle OR case; simplified to single FROM + if len(allowlist) > 1: + # Client-side filter fallback: search with SINCE only or ALL, then filter + if search_days is not None: + search_query = f'SINCE "{(datetime.utcnow() - timedelta(days=search_days)).strftime("%d-%b-%Y")}"' + else: + search_query = "ALL" + + logger.debug("IMAP search %s folder=%r criteria=%r", email, folder, search_query) + try: + typ, data = mail.search(None, search_query) + except imaplib.IMAP4.error as e: + logger.warning("IMAP search failed %s/%r: %s - trying ALL", email, folder, e) + typ, data = mail.search(None, "ALL") + if typ != "OK" or not data or not data[0]: + logger.info("IMAP search empty %s/%r", email, folder) + return [] + uids = data[0].split() + if not uids: + return [] + # Newest first: reverse UID list (UIDs increase) + uids = list(reversed(uids)) + # Respect max_mails (per account) + if len(uids) > max_mails: + uids = uids[:max_mails] + + # Client-side allowlist filtering if we used ALL fallback + # We'll still fetch and filter by From header later in parser, but to avoid fetching too many, + # we can peek headers first for From? Simpler: fetch all and let parser filter. + # For >1 allowlist and fallback ALL, we will filter after fetch via From header check. + + # Batch fetch with reconnect every 25 + batch_size = 25 + fetched = 0 + for i in range(0, len(uids), batch_size): + batch = uids[i : i + batch_size] + # reconnect every batch after first to avoid Bridge drops + if i > 0: + try: + mail.logout() + except Exception: + pass + time.sleep(0.3) + try: + mail = connect_imap(account) + mail.select(folder, readonly=True) + except Exception as e: + logger.warning("IMAP reconnect failed %s batch %d: %s", email, i, e) + break + + for uid in batch: + try: + uid_str = uid.decode() if isinstance(uid, bytes) else str(uid) + typ, fdata = mail.fetch(uid_str, "(BODY.PEEK[])") + if typ != "OK" or not fdata: + continue + # fdata is list of tuples; extract raw bytes + for item in fdata: + if isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], bytes): + raw_list.append(item[1]) + fetched += 1 + elif isinstance(item, bytes) and len(item) > 100: + raw_list.append(item) + fetched += 1 + except imaplib.IMAP4.error as e: + logger.debug("IMAP fetch uid %s failed %s: %s", uid.decode(errors="ignore"), email, e) + continue + except Exception as e: + logger.warning("IMAP fetch unexpected %s: %s", email, e) + continue + + logger.info("IMAP fetch done %s/%r: requested %d, fetched %d raw", email, folder, len(uids), len(raw_list)) + return raw_list + + except Exception as e: + logger.warning("IMAP fetch_raw_emails failed %s/%r: %s", email, folder, e, exc_info=True) + return [] + finally: + if mail is not None: + try: + mail.logout() + except Exception: + try: + mail.close() + except Exception: + pass diff --git a/src/advisory_rss/cert_tr/parser.py b/src/advisory_rss/cert_tr/parser.py new file mode 100644 index 0000000..f937463 --- /dev/null +++ b/src/advisory_rss/cert_tr/parser.py @@ -0,0 +1,447 @@ +"""Parse SGB/CERT-TR Proton Mail messages into NormalizedAdvisory. + +Handles: + - RFC2047 encoded headers (Subject, From) + - quoted-printable / base64 + charset windows-1254 fallback + - CVE/CWE/product extraction from HTML or plain body + - CVE-dedup ID: CERT-TR-CVE-YYYY-NNNNN or fallback to Message-Id hash + - Never marks as read (parser only) +""" +# ruff: noqa: BLE001, S112, SIM102 + +from __future__ import annotations + +import hashlib +import html +import logging +import re +from datetime import UTC, datetime +from email import policy +from email.header import decode_header +from email.message import EmailMessage +from email.parser import BytesParser +from email.utils import parsedate_to_datetime +from typing import Any + +from advisory_rss.cache.models import NormalizedAdvisory + +logger = logging.getLogger(__name__) + +CVE_RE = re.compile(r"CVE-\d{4}-\d{4,7}", re.IGNORECASE) +CWE_RE = re.compile(r"CWE-\d+", re.IGNORECASE) +# product heuristic: “pardus-etap-settings” ürünü -> group 1 +PRODUCT_RE = re.compile(r"[“\"']([^”\"']{2,80})[”\"']\s*ürün", re.IGNORECASE) +# fallback: quoted strings list +QUOTED_RE = re.compile(r"[“\"]([^”\"]{2,80})[”\"]") + +# Sender allowlist check is done outside but parser also extracts domain + + +def decode_rfc2047(s: str | None) -> str: + if not s: + return "" + try: + parts = decode_header(s) + out = "" + for raw, enc in parts: + if isinstance(raw, bytes): + try: + out += raw.decode(enc or "utf-8", errors="replace") + except LookupError: + out += raw.decode("utf-8", errors="replace") + else: + out += raw + return out + except Exception as e: + logger.debug("decode_rfc2047 failed %r: %s", s[:80], e) + return str(s) + + +def _extract_domain(from_header: str) -> str: + # from_header already decoded may contain + m = re.search(r"<([^>]+)>", from_header or "") + addr = m.group(1) if m else (from_header or "") + addr = addr.strip().lower() + if "@" in addr: + return addr.split("@", 1)[1].lower() + return addr + + +def _parse_date(date_str: str | None) -> datetime | None: + if not date_str: + return None + try: + dt = parsedate_to_datetime(date_str) + if dt and dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt + except Exception as e: + logger.debug("date parse failed %r: %s", date_str, e) + return None + + +def _html_to_text(html_body: str) -> str: + # Lightweight html -> text: strip tags, decode entities, keep line breaks + # Replace
,

,

with newlines before stripping + if not html_body: + return "" + # normalize quoted-printable already decoded to string + s = html_body + # Replace block tags with newline + s = re.sub(r"(?i)<\s*br\s*/?\s*>", "\n", s) + s = re.sub(r"(?i)", "\n\n", s) + s = re.sub(r"(?i)", "\n", s) + # Remove all tags + s = re.sub(r"<[^>]+>", " ", s) + # Decode html entities + s = html.unescape(s) + # Collapse whitespace but keep single newlines + s = s.replace("\r", "") + # Collapse multiple spaces + s = re.sub(r"[ \t]+", " ", s) + s = re.sub(r"\n{3,}", "\n\n", s) + s = re.sub(r" *\n *", "\n", s) + return s.strip() + + +def _extract_body(msg: EmailMessage) -> tuple[str, str | None]: + """Return (plain_text, html_raw_or_None). Decodes transfer encodings.""" + # Walk parts; prefer html then plain; handle multipart + plain: str | None = None + html_raw: str | None = None + + if msg.is_multipart(): + for part in msg.walk(): + ctype = part.get_content_type() + # skip containers + if ctype in ("multipart/alternative", "multipart/mixed", "multipart/related"): + continue + disp = (part.get_content_disposition() or "").lower() + if disp == "attachment": + continue + payload = part.get_payload(decode=True) # type: ignore[assignment] + if payload is None: + continue + charset = part.get_content_charset() or "utf-8" + # try charset fallback chain + text = None + assert isinstance(payload, (bytes, bytearray)) + for enc in (charset, "utf-8", "windows-1254", "iso-8859-9", "iso-8859-1"): + try: + text = payload.decode(enc, errors="replace") # type: ignore[union-attr] + break + except LookupError: + continue + except Exception: # noqa: BLE001 + continue + if text is None: + text = payload.decode("utf-8", errors="replace") # type: ignore[union-attr] + if ctype == "text/html" and html_raw is None: + html_raw = text + elif ctype == "text/plain" and plain is None: + plain = text + # fallback: if html but no plain, convert html to plain + if html_raw and not plain: + plain = _html_to_text(html_raw) + if plain and not html_raw: + # keep html_raw None + pass + return plain or "", html_raw + else: + ctype = msg.get_content_type() + payload = msg.get_payload(decode=True) # type: ignore[assignment] + if payload is None: + # get_payload may return str + payload_str = msg.get_payload() + if isinstance(payload_str, str): + # quoted-printable already? + return payload_str, None + return "", None + charset = msg.get_content_charset() or "utf-8" + text = None + assert isinstance(payload, (bytes, bytearray)) + for enc in (charset, "utf-8", "windows-1254", "iso-8859-9"): + try: + text = payload.decode(enc, errors="replace") # type: ignore[union-attr] + break + except LookupError: + continue + if text is None: + text = payload.decode("utf-8", errors="replace") # type: ignore[union-attr] + if ctype == "text/html": + return _html_to_text(text), text + return text, None + + +def _extract_cves(text: str) -> list[str]: + found = CVE_RE.findall(text or "") + # normalize upper + norm = [c.upper() for c in found] + # dedup preserve order + seen = set() + out = [] + for c in norm: + if c not in seen: + seen.add(c) + out.append(c) + return out + + +def _extract_cwe(text: str) -> str | None: + m = CWE_RE.search(text or "") + return m.group(0).upper() if m else None + + +def _extract_product(text: str) -> str | None: + if not text: + return None + m = PRODUCT_RE.search(text) + if m: + prod = m.group(1).strip() + # cleanup quotes artifacts + prod = prod.strip("“”\"' ") + if prod and len(prod) > 1: + return prod + # fallback: find quoted strings containing dash or lowercase + not CWE/CVE + quoted = QUOTED_RE.findall(text) + candidates = [] + for q in quoted: + q = q.strip() + if not q or q.upper().startswith("CWE") or q.upper().startswith("CVE"): + continue + if "TÜBİTAK" in q or "Siber" in q or len(q) > 40: + continue + # product usually contains - or _ + if "-" in q or "_" in q or q.islower() or "/" in q: + candidates.append(q) + if candidates: + # shortest that looks like package name + candidates.sort(key=len) + return candidates[0].strip() + return None + + +def _build_summary(subject_decoded: str, product: str | None, cve: str | None, cwe: str | None) -> str: + # If product + CVE, build canonical summary, else decoded subject + if product and cve: + base = f"{product}: {cve}" + if cwe: + base += f" ({cwe})" + return base + if cve and cwe: + return f"{cve} - {cwe}" + if cve: + return f"Zafiyet Bildirimi {cve}" + # fallback to subject trimmed + s = (subject_decoded or "").strip() + if s: + # Remove excessive prefix "Zafiyet Bildiriminiz Hakkında" + s = re.sub(r"^\s*Zafiyet Bildiriminiz Hakkında\s*[:\-]?\s*", "", s, flags=re.IGNORECASE) + s = s.strip(" ()") + if s: + return s + return subject_decoded.strip() or "CERT-TR Zafiyet Bildirimi" + + +def parse_cert_tr_email(raw_bytes: bytes) -> NormalizedAdvisory | None: + """Parse raw RFC822 bytes from Proton Bridge into NormalizedAdvisory. + + Returns None if sender not in allowlist or no CVE and not enough data. + Uses CVE-based id for dedup (CERT-TR-CVE-...), else Message-Id hash. + Never marks mail read. + """ + try: + msg = BytesParser(policy=policy.default).parsebytes(raw_bytes) + except Exception as e: + logger.warning("Failed to parse email bytes: %s", e) + return None + + # Headers + subject_raw = msg.get("Subject") or "" + subject = decode_rfc2047(str(subject_raw)) + from_raw = msg.get("From") or "" + from_decoded = decode_rfc2047(str(from_raw)) + reply_to = decode_rfc2047(str(msg.get("Reply-To") or "")) + message_id = (msg.get("Message-Id") or msg.get("Message-ID") or "").strip() + # also X-Pm-External-Id fallback + if not message_id: + message_id = (msg.get("X-Pm-External-Id") or "").strip() + date_hdr = msg.get("Date") or msg.get("X-Pm-Date") or "" + date_dt = _parse_date(str(date_hdr)) + + # Body + plain_text, html_raw = _extract_body(msg) + # Combine subject + plain for CVE search (subject often contains CVE) + combined_text = f"{subject}\n{plain_text}" + if html_raw: + combined_text += f"\n{html_raw}" + + cves = _extract_cves(combined_text) + cve_primary = cves[0] if cves else None + cwe = _extract_cwe(combined_text) + product = _extract_product(combined_text) + + # If no CVE found, still maybe useful? Require at least product or cwe or cve + # For strict CERT-TR, require CVE; otherwise skip to avoid noise from unrelated mail + if not cve_primary: + # also check message_id contains CVE? + if message_id: + m = CVE_RE.search(message_id) + if m: + cve_primary = m.group(0).upper() + if cve_primary not in cves: + cves.append(cve_primary) + + if not cve_primary and not product and not cwe: + logger.debug("Skipping non-CERT-TR mail subject=%r from=%r", subject[:60], from_decoded[:60]) + return None + # If still no CVE, we cannot make stable id from CVE; use message-id hash but mark as cert-tr generic + # For now require CVE to produce advisory; if no CVE, create id from hash but keep cve_id None + # We will allow it but id will be hash-based + # Validation: sender domain check is done at source level, not here + + # ID generation: CERT-TR-CVE-xxx priority + if cve_primary: + ghsa_id = f"CERT-TR-{cve_primary.upper()}" + else: + # hash message-id + date + subject + h = hashlib.sha256() + h.update((message_id or subject or plain_text[:100]).encode("utf-8", errors="ignore")) + digest = h.hexdigest()[:12].upper() + ghsa_id = f"CERT-TR-MSG-{digest}" + + # Extra CVEs beyond primary + extra_cves = [c for c in cves if c != cve_primary] + + # Summary & description + summary = _build_summary(subject, product, cve_primary, cwe) + # Description: plain_text truncated 8000 chars for RSS, keep full + description = plain_text.strip() + if not description and html_raw: + description = _html_to_text(html_raw) + if not description: + description = combined_text[:2000] + # Include product/vendor context if product found and not in summary + # Keep within model limits; no hard truncation here, builder does it + + # References + refs: list[str] = [] + if cve_primary: + refs.append(f"https://nvd.nist.gov/vuln/detail/{cve_primary}") + refs.append(f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve_primary}") + # also include siberguvenlik portal (generic) + refs.append("https://siberguvenlik.gov.tr") + + # Links: html_url fallback to siberguvenlik + html_url = "" + if cve_primary: + # placeholder deep link not known; use NVD + html_url = f"https://nvd.nist.gov/vuln/detail/{cve_primary}" + else: + html_url = "https://siberguvenlik.gov.tr" + + # Severity: try to infer from text (kritik/yüksek/orta/düşük) + severity = None + low_text = (plain_text or "").lower() + if "kritik" in low_text or "critical" in low_text: + severity = "critical" + elif "yüksek" in low_text or "high" in low_text: + severity = "high" + elif "orta" in low_text or "medium" in low_text or "moderate" in low_text: + severity = "medium" + elif "düşük" in low_text or "low" in low_text: + severity = "low" + + # Package extraction: ecosystem maybe? + # infer ecosystem from product name: pardus => deb? keep None + package_ecosystem = None + package_name = product + + # State: CERT-TR mails are allocations; consider "published" but note awaiting patch + state = "published" + + # Updated/published = date + published_at = date_dt + updated_at = date_dt + created_at = date_dt + + # Author/publisher + author_login = from_decoded or "cve@siberguvenlik.gov.tr" + # Try to extract email addr only + m = re.search(r"<([^>]+@[^>]+)>", author_login) + if m: + author_login = m.group(1).strip() + publisher_login = "siberguvenlik.gov.tr" + + # identifiers list + identifiers = [] + for c in cves: + identifiers.append({"type": "CVE", "value": c}) + if cwe: + identifiers.append({"type": "CWE", "value": cwe}) + + # raw preservation (redacted password not in raw) + raw: dict[str, Any] = { + "message_id": message_id, + "subject": subject, + "from": from_decoded, + "reply_to": reply_to, + "date": str(date_hdr), + "cves": cves, + "cwe": cwe, + "product": product, + "html_raw_snippet": (html_raw[:2000] if html_raw else "")[:2000], + "plain_snippet": plain_text[:2000], + "x_pm_external_id": str(msg.get("X-Pm-External-Id") or ""), + "x_pm_internal_id": str(msg.get("X-Pm-Internal-Id") or ""), + } + + adv = NormalizedAdvisory( + ghsa_id=ghsa_id, + cve_id=cve_primary, + summary=summary, + description=description, + severity=severity, + package_ecosystem=package_ecosystem, + package_name=package_name, + vulnerable_version_range=None, + patched_versions=None, + affected_versions=None, + published_at=published_at, + updated_at=updated_at, + withdrawn_at=None, + created_at=created_at, + state=state, + html_url=html_url, + repository_url=None, + repository_full_name=f"cert-tr/{product}" if product else "cert-tr/siberguvenlik", + references=refs, + author_login=author_login, + publisher_login=publisher_login, + identifiers=identifiers, + raw=raw, + source="cert-tr", + extra_cves=extra_cves, + ) + logger.debug("Parsed CERT-TR mail ghsa_id=%s cve=%s product=%r", ghsa_id, cve_primary, product) + return adv + + +def is_cert_tr_sender(from_header: str, allowlist: list[str]) -> bool: + if not from_header: + return False + decoded = decode_rfc2047(from_header) + domain = _extract_domain(decoded) + if not domain: + # fallback raw lower + domain = from_header.lower() + for allowed in allowlist: + allowed = allowed.lower().strip() + if not allowed: + continue + if domain == allowed or domain.endswith("." + allowed) or allowed in domain: + return True + # also substring check for safety + if allowed in from_header.lower(): + return True + return False diff --git a/src/advisory_rss/cert_tr/source.py b/src/advisory_rss/cert_tr/source.py new file mode 100644 index 0000000..5ecd174 --- /dev/null +++ b/src/advisory_rss/cert_tr/source.py @@ -0,0 +1,111 @@ +"""CERT-TR source orchestrator: IMAP fetch -> parser -> NormalizedAdvisory list.""" +# ruff: noqa: BLE001 + +from __future__ import annotations + +import logging +from typing import Any + +from advisory_rss.cache.models import NormalizedAdvisory +from advisory_rss.cert_tr import imap as cert_imap +from advisory_rss.cert_tr.parser import is_cert_tr_sender, parse_cert_tr_email +from advisory_rss.config.settings import Settings + +logger = logging.getLogger(__name__) + + +class CertTrSource: + def __init__(self, settings: Settings): + self.settings = settings + + def fetch_all(self) -> tuple[list[NormalizedAdvisory], dict[str, Any]]: + """Fetch from all configured Proton accounts. + + Returns (advisories, diag). Never raises (errors go to diag). + Dedup by ghsa_id (CERT-TR-CVE-...) keep newest (by updated_at). + """ + diag: dict[str, Any] = { + "accounts": 0, + "raw_fetched": 0, + "parsed": 0, + "filtered_sender": 0, + "errors": [], + } + if not self.settings.enable_cert_tr: + logger.debug("CERT-TR disabled (ENABLE_CERT_TR=false)") + return [], diag + + accounts = self.settings.get_proton_accounts() + if not accounts: + msg = "ENABLE_CERT_TR=true but no PROTON_BRIDGE_EMAIL/EMAILS configured" + logger.warning(msg) + diag["errors"].append(msg) + return [], diag + + allowlist = self.settings.cert_tr_sender_list() + max_mails = self.settings.cert_tr_max_mails + search_days = self.settings.cert_tr_search_days + + all_advs: list[NormalizedAdvisory] = [] + per_account_max = max_mails + # If multiple accounts, distribute budget + if len(accounts) > 1: + per_account_max = max(20, max_mails // len(accounts)) + + for acc in accounts: + email = acc.get("email", "unknown") + folder = acc.get("folder", "INBOX") + # Validate credentials + if not acc.get("password"): + msg = f"Proton account {email}/{folder}: missing PROTON_BRIDGE_PASSWORD - skipping" + logger.warning(msg) + diag["errors"].append(msg) + continue + diag["accounts"] += 1 + try: + raw_list = cert_imap.fetch_raw_emails(acc, allowlist, max_mails=per_account_max, search_days=search_days) + except Exception as e: + msg = f"IMAP error {email}/{folder}: {e}" + logger.warning(msg) + diag["errors"].append(msg) + continue + diag["raw_fetched"] += len(raw_list) + + for raw in raw_list: + try: + adv = parse_cert_tr_email(raw) + if not adv: + continue + # Sender filter (allowlist) - parser already extracts from raw header via raw["from"] + # Need to re-check using parsed adv's author_login / raw sender + from_hdr = adv.raw.get("from") or adv.author_login or "" + if allowlist and not is_cert_tr_sender(from_hdr, allowlist): + diag["filtered_sender"] += 1 + continue + # Also filter by folder? No, folder already limits + all_advs.append(adv) + diag["parsed"] += 1 + except Exception as e: + logger.debug("parse_cert_tr_email failed: %s", e, exc_info=True) + continue + + # Dedup by ghsa_id keep newest updated_at + seen: dict[str, NormalizedAdvisory] = {} + for adv in all_advs: + prev = seen.get(adv.ghsa_id) + if prev is None or adv.sort_key() > prev.sort_key(): + seen[adv.ghsa_id] = adv + + deduped = list(seen.values()) + deduped.sort(key=lambda a: a.sort_key(), reverse=True) + + logger.info( + "CERT-TR fetch done accounts=%d raw=%d parsed=%d deduped=%d filtered_sender=%d errors=%d", + diag["accounts"], + diag["raw_fetched"], + diag["parsed"], + len(deduped), + diag["filtered_sender"], + len(diag["errors"]), + ) + return deduped, diag diff --git a/src/advisory_rss/cli.py b/src/advisory_rss/cli.py index 1f309cb..89b2de9 100644 --- a/src/advisory_rss/cli.py +++ b/src/advisory_rss/cli.py @@ -1,5 +1,6 @@ from __future__ import annotations +# ruff: noqa: BLE001, S110 import asyncio import getpass import logging @@ -121,90 +122,172 @@ def auth_status() -> None: @cli.command() -def sync() -> None: +@click.option( + "--source", + type=click.Choice(["all", "github", "cert-tr"], case_sensitive=False), + default="all", + show_default=True, + help="Which source to sync (all includes GitHub + CERT-TR)", +) +def sync(source: str = "all") -> None: settings = get_settings() _setup_logging(settings.log_level, log_file=settings.log_file, log_format=settings.log_format) logger.info( - "sync started", + "sync started source=%s", + source, extra={ "filter_mode": settings.filter_mode, "cache_path": str(settings.resolved_cache_path), + "enable_cert_tr": settings.enable_cert_tr, }, ) + source = source.lower() token = load_token(settings) - if not token: - click.echo("No token set. Set GITHUB_TOKEN in .env or run `app auth login`.", err=True) - sys.exit(1) + # Allow cert-tr only sync without GitHub token + need_github = source in ("all", "github") + if need_github and not token: + # If CERT-TR is enabled and user asked all, still allow cert-tr part to run + if settings.enable_cert_tr and source == "all": + click.echo("Warning: GITHUB_TOKEN not set - GitHub sync will be skipped, CERT-TR only.", err=True) + logger.warning("sync without GitHub token - GitHub skipped, CERT-TR only") + need_github = False + else: + click.echo("No token set. Set GITHUB_TOKEN in .env or run `app auth login`.", err=True) + sys.exit(1) cache = CacheStore(settings.resolved_cache_path) async def _run() -> int: - client = GitHubClient(settings, token, cache=cache) - try: - advs, diag = await client.sync_all(filter_mode=settings.filter_mode) - count = cache.upsert_advisories(advs) - cache.mark_success(user=diag.get("authenticated_login")) - nxt = (datetime.now(UTC) + timedelta(seconds=settings.refresh_interval)).isoformat() - cache.set_meta("next_scheduled_sync", nxt) - logger.info( - "sync ok: upserted=%d filtered=%s raw=%s repos=%s orgs=%s", - count, - diag.get("filtered_count"), - diag.get("raw_count"), - diag.get("repos_scanned"), - diag.get("orgs_scanned"), - extra={"diag": diag}, - ) - click.echo( - f"Sync OK - {count} advisories upserted (filtered {diag.get('filtered_count')} from {diag.get('raw_count')} raw, {diag.get('repos_scanned')} repos, {diag.get('orgs_scanned')} orgs)" - ) - if diag.get("errors"): - click.echo(f"Warnings ({len(diag['errors'])}):", err=True) - for e in diag["errors"][:5]: - click.echo(f" - {_redact(e)}", err=True) - for e in diag["errors"]: - logger.warning("sync warning: %s", _redact(e)) - if advs: - click.echo(f"Latest: {advs[0].ghsa_id} - {advs[0].summary[:80]}") - logger.debug("latest advisory: %s - %s", advs[0].ghsa_id, advs[0].summary[:80]) - else: - click.echo( - "No advisories matched filter - cache remains empty. Check FILTER_MODE or author login." - ) - logger.warning( - "sync produced no advisories (filter=%s login=%s)", - settings.filter_mode, - diag.get("authenticated_login"), + all_advisories = [] + combined_diag: dict = { + "github": None, + "cert_tr": None, + "errors": [], + } + # --- GitHub --- + if need_github and token: + client = GitHubClient(settings, token, cache=cache) + try: + advs, diag = await client.sync_all(filter_mode=settings.filter_mode) + all_advisories.extend(advs) + combined_diag["github"] = diag + combined_diag["errors"].extend(diag.get("errors") or []) + logger.info( + "GitHub sync ok: upserted=%d filtered=%s raw=%s repos=%s orgs=%s", + len(advs), + diag.get("filtered_count"), + diag.get("raw_count"), + diag.get("repos_scanned"), + diag.get("orgs_scanned"), + extra={"diag": diag}, ) - return count - except AuthError as e: - msg = _redact(str(e)) - logger.error("sync auth error: %s", msg, exc_info=True) - click.echo(f"Auth error (401): {msg}", err=True) - cache.mark_error(msg) - sys.exit(1) - except RateLimitError as e: - msg = _redact(str(e)) - logger.warning("sync rate limited retry_after=%s: %s", e.retry_after, msg) - click.echo( - f"Rate limited - retry after {e.retry_after}: {msg}", - err=True, - ) - if e.retry_after: - cache.set_meta( - "rate_limited_until", - (datetime.now(UTC) + timedelta(seconds=e.retry_after)).isoformat(), + click.echo( + f"GitHub sync OK - {len(advs)} advisories (filtered {diag.get('filtered_count')} from {diag.get('raw_count')} raw, {diag.get('repos_scanned')} repos)" ) - cache.mark_error(msg) - sys.exit(1) - except (OSError, ValueError, RuntimeError, httpx.HTTPError) as e: - msg = _redact(str(e)) - logger.error("sync failed: %s", msg, exc_info=True) - click.echo(f"Sync failed: {msg}", err=True) - cache.mark_error(msg) - sys.exit(1) - finally: - await client.close() + if advs: + click.echo(f"Latest GitHub: {advs[0].ghsa_id} - {advs[0].summary[:80]}") + except AuthError as e: + msg = _redact(str(e)) + logger.error("GitHub sync auth error: %s", msg, exc_info=True) + click.echo(f"GitHub Auth error (401): {msg}", err=True) + cache.mark_error(msg) + if source == "github": + sys.exit(1) + combined_diag["errors"].append(msg) + except RateLimitError as e: + msg = _redact(str(e)) + logger.warning("GitHub rate limited retry_after=%s: %s", e.retry_after, msg) + click.echo(f"GitHub rate limited - retry after {e.retry_after}: {msg}", err=True) + if e.retry_after: + cache.set_meta( + "rate_limited_until", + (datetime.now(UTC) + timedelta(seconds=e.retry_after)).isoformat(), + ) + cache.mark_error(msg) + if source == "github": + sys.exit(1) + combined_diag["errors"].append(msg) + except (OSError, ValueError, RuntimeError, httpx.HTTPError) as e: + msg = _redact(str(e)) + logger.error("GitHub sync failed: %s", msg, exc_info=True) + click.echo(f"GitHub sync failed: {msg}", err=True) + cache.mark_error(msg) + if source == "github": + sys.exit(1) + combined_diag["errors"].append(msg) + finally: + await client.close() + elif source in ("all", "github"): + click.echo("GitHub sync skipped (no token).") + + # --- CERT-TR (Proton) --- + if source in ("all", "cert-tr") and settings.enable_cert_tr: + try: + from advisory_rss.cert_tr.source import CertTrSource + + src = CertTrSource(settings) + advs_ct, diag_ct = src.fetch_all() + combined_diag["cert_tr"] = diag_ct + combined_diag["errors"].extend(diag_ct.get("errors") or []) + if advs_ct: + all_advisories.extend(advs_ct) + click.echo(f"CERT-TR sync OK - {len(advs_ct)} advisories from {diag_ct.get('accounts')} account(s) (raw {diag_ct.get('raw_fetched')})") + click.echo(f"Latest CERT-TR: {advs_ct[0].ghsa_id} - {advs_ct[0].summary[:80]}") + else: + click.echo(f"CERT-TR sync: no advisories (raw {diag_ct.get('raw_fetched')}, filtered_sender {diag_ct.get('filtered_sender')})") + if diag_ct.get("errors"): + for e in diag_ct["errors"][:3]: + click.echo(f" CERT-TR warn: {_redact(e)}", err=True) + except (OSError, ValueError, RuntimeError) as e: + msg = _redact(str(e)) + logger.error("CERT-TR sync failed: %s", msg, exc_info=True) + click.echo(f"CERT-TR sync failed: {msg}", err=True) + cache.mark_error(msg) + if source == "cert-tr": + sys.exit(1) + combined_diag["errors"].append(msg) + elif source in ("all", "cert-tr") and not settings.enable_cert_tr: + click.echo("CERT-TR disabled (ENABLE_CERT_TR=false) - skipping.", err=True) + + # --- Upsert combined --- + if not all_advisories: + # Check if both sources were skipped vs empty result + click.echo("No advisories from any source - cache unchanged. Check GITHUB_TOKEN / PROTON_* settings.") + # Still mark success if no error (to update next sync) + if not combined_diag["errors"]: + cache.mark_success(user=cache.get_meta("authenticated_user")) + nxt = (datetime.now(UTC) + timedelta(seconds=settings.refresh_interval)).isoformat() + cache.set_meta("next_scheduled_sync", nxt) + else: + cache.mark_error("; ".join(combined_diag["errors"][:3])) + return 0 + + # Dedup across sources (GitHub GHSA-* vs CERT-TR-* no collision, but keep logic) + # CERT-TR already deduped by CVE; GitHub deduped too; cross-source no overlap + count = cache.upsert_advisories(all_advisories) + # keep authenticated_user from GitHub if present, else cert-tr + user = None + if combined_diag.get("github"): + user = combined_diag["github"].get("authenticated_login") + if not user: + user = cache.get_meta("authenticated_user") + cache.mark_success(user=user) + nxt = (datetime.now(UTC) + timedelta(seconds=settings.refresh_interval)).isoformat() + cache.set_meta("next_scheduled_sync", nxt) + # Store cert-tr diag for health + try: + import json + + if combined_diag.get("cert_tr"): + cache.set_meta("cert_tr_diag", json.dumps(combined_diag["cert_tr"], ensure_ascii=False)) + except Exception: + pass + click.echo(f"Sync OK - {count} advisories upserted total ({len(all_advisories)} fetched before dedup)") + if combined_diag["errors"]: + click.echo(f"Warnings ({len(combined_diag['errors'])}):", err=True) + for e in combined_diag["errors"][:5]: + click.echo(f" - {_redact(e)}", err=True) + return count asyncio.run(_run()) @@ -530,6 +613,15 @@ def _status() -> None: click.echo(f"Bind address: {settings.effective_bind_address()}:{settings.port}") click.echo(f"Cache path: {settings.resolved_cache_path}") click.echo(f"Cached advisories: {meta.advisories_count}") + # Per-source breakdown + try: + all_adv = cache.load_all() + github_c = sum(1 for a in all_adv if (getattr(a, "source", "github") or "github") == "github") + cert_c = sum(1 for a in all_adv if getattr(a, "source", "") == "cert-tr") + click.echo(f" - GitHub: {github_c}") + click.echo(f" - CERT-TR: {cert_c}") + except Exception: + pass click.echo(f"Last sync: {meta.last_successful_sync or 'never'}") click.echo(f"Next sync: {meta.next_scheduled_sync or 'not scheduled'}") click.echo(f"Rate limited until: {meta.rate_limited_until or 'no'}") @@ -539,6 +631,18 @@ def _status() -> None: click.echo(f"Filter mode: {settings.filter_mode}") click.echo(f"Refresh interval: {settings.refresh_interval}s") click.echo(f"Max items (RSS): {settings.max_items}") + click.echo(f"CERT-TR enabled: {settings.enable_cert_tr}") + if settings.enable_cert_tr: + accs = settings.get_proton_accounts() + click.echo(f"CERT-TR accounts: {len(accs)}") + for a in accs: + # Never show password + click.echo(f" - {a['email']} -> folder={a['folder']} host={a['host']}:{a['port']} sec={a['security']}") + click.echo(f"CERT-TR allowlist: {settings.cert_tr_sender_allowlist}") + click.echo(f"CERT-TR max mails: {settings.cert_tr_max_mails} search_days: {settings.cert_tr_search_days or 'all'}") + diag_raw = cache.get_meta("cert_tr_diag") + if diag_raw: + click.echo(f"CERT-TR last diag: {diag_raw[:300]}") # Also try to infer loopback status from advisory_rss.server.bind import is_loopback @@ -546,8 +650,10 @@ def _status() -> None: click.echo(f"Bind is loopback: {bind_ok}") if not bind_ok: click.echo("WARNING: bind address is NOT loopback - will refuse to start", err=True) - if not token: + if not token and not settings.enable_cert_tr: click.echo("Hint: set GITHUB_TOKEN in .env or run `app auth login`", err=True) + elif not token: + click.echo("Note: GITHUB_TOKEN not set - GitHub source disabled, CERT-TR only", err=True) # Also support `app auth login` already; add root-level `app login` alias for convenience diff --git a/src/advisory_rss/config/constants.py b/src/advisory_rss/config/constants.py index da378d0..2664cd3 100644 --- a/src/advisory_rss/config/constants.py +++ b/src/advisory_rss/config/constants.py @@ -30,5 +30,16 @@ GITHUB_API_VERSION = "2022-11-28" # Pinned to 2022-11-28 (stable). 2026-03-10 also valid but not required. +# Cert-TR / Proton +DEFAULT_CERT_TR_ENABLED = False +DEFAULT_PROTON_IMAP_HOST = "127.0.0.1" +DEFAULT_PROTON_IMAP_PORT = 1143 +DEFAULT_PROTON_IMAP_SECURITY = "STARTTLS" +DEFAULT_PROTON_FOLDER = "INBOX" +DEFAULT_CERT_TR_SENDER_ALLOWLIST = "siberguvenlik.gov.tr" + # Security TOKEN_REDACT_PATTERN = r"(gh[pousr]_[A-Za-z0-9_-]+|github_pat_[A-Za-z0-9_-]+)" + +# Proton bridge password redaction (append to generic pattern via separate check) +PROTON_PASSWORD_HINT = "proton" diff --git a/src/advisory_rss/config/settings.py b/src/advisory_rss/config/settings.py index 45d9be4..089cdf5 100644 --- a/src/advisory_rss/config/settings.py +++ b/src/advisory_rss/config/settings.py @@ -11,6 +11,8 @@ from advisory_rss.config.constants import ( DEFAULT_BIND_ADDRESS, DEFAULT_CACHE_PATH, + DEFAULT_CERT_TR_ENABLED, + DEFAULT_CERT_TR_SENDER_ALLOWLIST, DEFAULT_FILTER_MODE, DEFAULT_GITHUB_API_BASE, DEFAULT_LOG_FORMAT, @@ -19,6 +21,10 @@ DEFAULT_MAX_PAGES_PER_REPO, DEFAULT_MAX_REPOS, DEFAULT_PORT, + DEFAULT_PROTON_FOLDER, + DEFAULT_PROTON_IMAP_HOST, + DEFAULT_PROTON_IMAP_PORT, + DEFAULT_PROTON_IMAP_SECURITY, DEFAULT_REFRESH_INTERVAL, ) from advisory_rss.server.bind import validate_bind_address @@ -97,6 +103,46 @@ def __init__( default=DEFAULT_MAX_PAGES_PER_REPO, validation_alias="MAX_PAGES_PER_REPO" ) + # Cert-TR / Proton Mail (multi-account, per-mailbox folder) + enable_cert_tr: bool = Field( + default=DEFAULT_CERT_TR_ENABLED, validation_alias="ENABLE_CERT_TR" + ) + proton_bridge_email: str | None = Field(default=None, validation_alias="PROTON_BRIDGE_EMAIL") + proton_bridge_password: str | None = Field( + default=None, validation_alias="PROTON_BRIDGE_PASSWORD" + ) + proton_bridge_host: str = Field( + default=DEFAULT_PROTON_IMAP_HOST, validation_alias="PROTON_BRIDGE_HOST" + ) + proton_imap_port: int = Field( + default=DEFAULT_PROTON_IMAP_PORT, validation_alias="PROTON_IMAP_PORT" + ) + proton_imap_security: str = Field( + default=DEFAULT_PROTON_IMAP_SECURITY, validation_alias="PROTON_IMAP_SECURITY" + ) + proton_imap_folder: str = Field( + default=DEFAULT_PROTON_FOLDER, validation_alias="PROTON_IMAP_FOLDER" + ) + # Multi-account overrides (comma/space separated) + proton_bridge_emails: str | None = Field( + default=None, validation_alias="PROTON_BRIDGE_EMAILS" + ) + proton_bridge_passwords: str | None = Field( + default=None, validation_alias="PROTON_BRIDGE_PASSWORDS" + ) + proton_imap_folders: str | None = Field( + default=None, validation_alias="PROTON_IMAP_FOLDERS" + ) + # Optional: per-account JSON-like override not needed; plural fields suffice + cert_tr_sender_allowlist: str = Field( + default=DEFAULT_CERT_TR_SENDER_ALLOWLIST, + validation_alias="CERT_TR_SENDER_ALLOWLIST", + ) + cert_tr_max_mails: int = Field(default=200, validation_alias="CERT_TR_MAX_MAILS") + cert_tr_search_days: int | None = Field( + default=None, validation_alias="CERT_TR_SEARCH_DAYS" + ) + def extra_repo_list(self) -> list[str]: import logging @@ -226,6 +272,62 @@ def _validate_log_format(cls, v: str) -> str: raise ValueError(f"LOG_FORMAT must be 'text' or 'json' (got {v!r})") return fmt + @field_validator("proton_imap_security") + @classmethod + def _validate_proton_security(cls, v: str) -> str: + lvl = (v or DEFAULT_PROTON_IMAP_SECURITY).strip().upper() + if lvl not in {"STARTTLS", "SSL", "NONE"}: + raise ValueError("PROTON_IMAP_SECURITY must be STARTTLS|SSL|NONE") + return lvl + + @field_validator("proton_imap_folder") + @classmethod + def _validate_proton_folder(cls, v: str) -> str: + fv = (v or DEFAULT_PROTON_FOLDER).strip() + if not fv: + return DEFAULT_PROTON_FOLDER + # Allow IMAP hierarchy chars: letters, digits, /, ., -, _, space + import re + + if len(fv) > 100: + raise ValueError("PROTON_IMAP_FOLDER too long") + if not re.match(r"^[\w .\-/]+$", fv): + raise ValueError(f"PROTON_IMAP_FOLDER {fv!r} contains invalid chars") + return fv + + @field_validator("proton_imap_folders") + @classmethod + def _validate_proton_folders(cls, v: str | None) -> str | None: + if v is None or not str(v).strip(): + return v + import re + + raw = str(v).strip() + if len(raw) > 500: + raise ValueError("PROTON_IMAP_FOLDERS too long") + # split and validate each + parts = [p.strip() for p in re.split(r"[,\s;]+", raw) if p.strip()] + for p in parts: + if len(p) > 100 or not re.match(r"^[\w .\-/]+$", p): + raise ValueError(f"PROTON_IMAP_FOLDERS entry {p!r} invalid") + return v + + @field_validator("cert_tr_max_mails") + @classmethod + def _validate_cert_tr_max(cls, v: int) -> int: + if not (1 <= v <= 5000): + raise ValueError("CERT_TR_MAX_MAILS must be 1..5000") + return v + + @field_validator("cert_tr_search_days") + @classmethod + def _validate_search_days(cls, v: int | None) -> int | None: + if v is None: + return None + if not (0 <= v <= 3650): + raise ValueError("CERT_TR_SEARCH_DAYS must be 0..3650 or empty (all)") + return v + @field_validator("log_file") @classmethod def _validate_log_file(cls, v: str | None) -> str | None: @@ -245,6 +347,87 @@ def _validate_log_file(cls, v: str | None) -> str | None: ) return str(v).strip() + def _split_list(self, raw: str | None) -> list[str]: + if not raw or not raw.strip(): + return [] + # comma, whitespace or semicolon separated, keep non-empty + import re + + parts = re.split(r"[,\s;]+", raw.strip()) + return [p.strip() for p in parts if p.strip()] + + def get_proton_accounts(self) -> list[dict[str, str]]: + """Resolve multi-account Proton config. + + Supports: + - Single: PROTON_BRIDGE_EMAIL + PROTON_BRIDGE_PASSWORD + PROTON_IMAP_FOLDER + - Multi: PROTON_BRIDGE_EMAILS= a@proton.me,b@proton.me + PROTON_BRIDGE_PASSWORDS= pass1,pass2 + PROTON_IMAP_FOLDERS= INBOX,INBOX.CERT-TR + Indices aligned; missing values fallback to single/defaults. + Each account is {email, password, folder, host, port, security}. + Folder defaults to PROTON_IMAP_FOLDER / INBOX if not per-account. + """ + emails: list[str] = [] + passwords: list[str] = [] + folders: list[str] = [] + + if self.proton_bridge_emails and self.proton_bridge_emails.strip(): + emails = self._split_list(self.proton_bridge_emails) + if self.proton_bridge_passwords: + passwords = self._split_list(self.proton_bridge_passwords) + if self.proton_imap_folders: + folders = self._split_list(self.proton_imap_folders) + elif self.proton_bridge_email and self.proton_bridge_email.strip(): + emails = [self.proton_bridge_email.strip()] + if self.proton_bridge_password: + passwords = [self.proton_bridge_password.strip()] + # folder for single: use explicit plural if set else singular + if self.proton_imap_folders and self.proton_imap_folders.strip(): + folders = self._split_list(self.proton_imap_folders) + else: + folders = [self.proton_imap_folder.strip() or DEFAULT_PROTON_FOLDER] + else: + return [] + + # Align lengths: pad passwords/folders with defaults + out: list[dict[str, str]] = [] + default_folder = (self.proton_imap_folder or DEFAULT_PROTON_FOLDER).strip() or DEFAULT_PROTON_FOLDER + # If single password provided but multiple emails, reuse it + if len(passwords) == 1 and len(emails) > 1: + passwords = passwords * len(emails) + for idx, email in enumerate(emails): + if not email or "@" not in email: + continue + pw = "" + if idx < len(passwords): + pw = passwords[idx] + elif passwords: + pw = passwords[0] + elif self.proton_bridge_password: + pw = self.proton_bridge_password + folder = default_folder + if idx < len(folders) and folders[idx]: + folder = folders[idx] + # Normalize folder: strip but keep case (IMAP case-sensitive usually) + folder = folder.strip() or default_folder + out.append( + { + "email": email.strip(), + "password": pw.strip() if pw else "", + "folder": folder, + "host": (self.proton_bridge_host or DEFAULT_PROTON_IMAP_HOST).strip(), + "port": str(self.proton_imap_port), + "security": (self.proton_imap_security or DEFAULT_PROTON_IMAP_SECURITY).strip().upper(), + } + ) + return out + + def cert_tr_sender_list(self) -> list[str]: + if not self.cert_tr_sender_allowlist or not self.cert_tr_sender_allowlist.strip(): + return [DEFAULT_CERT_TR_SENDER_ALLOWLIST] + return [s.strip().lower() for s in self._split_list(self.cert_tr_sender_allowlist) if s.strip()] + def effective_bind_address(self) -> str: if self.host and self.bind_address == DEFAULT_BIND_ADDRESS: validate_bind_address(self.host) diff --git a/src/advisory_rss/rss/builder.py b/src/advisory_rss/rss/builder.py index 317e9be..255af27 100644 --- a/src/advisory_rss/rss/builder.py +++ b/src/advisory_rss/rss/builder.py @@ -63,9 +63,12 @@ def _build_item_xml(adv: NormalizedAdvisory) -> str: # Title logic severity = _severity_label(adv.severity) is_withdrawn = bool(adv.withdrawn_at) or adv.state.lower() == "withdrawn" + src = getattr(adv, "source", "github") or "github" prefix = "" if is_withdrawn: prefix = "[WITHDRAWN] " + elif src == "cert-tr": + prefix = "[CERT-TR] " elif severity not in ("unknown", "n/a"): prefix = f"[{severity.upper()}] " base_title = adv.summary.strip() or adv.ghsa_id @@ -73,7 +76,12 @@ def _build_item_xml(adv: NormalizedAdvisory) -> str: title_esc = escape_text(title) raw_link = adv.html_url.strip() if adv.html_url else "" - link = raw_link if _is_safe_url(raw_link) else f"https://github.com/advisories/{adv.ghsa_id}" + if src == "cert-tr": + # fallback to NVD or siberguvenlik portal if not safe + fallback = f"https://nvd.nist.gov/vuln/detail/{adv.cve_id}" if adv.cve_id else "https://siberguvenlik.gov.tr" + link = raw_link if _is_safe_url(raw_link) else fallback + else: + link = raw_link if _is_safe_url(raw_link) else f"https://github.com/advisories/{adv.ghsa_id}" link_esc = escape_text(link) guid = escape_text(adv.ghsa_id) @@ -85,6 +93,7 @@ def _build_item_xml(adv: NormalizedAdvisory) -> str: author = escape_text(adv.author_login or "unknown") # categories cats: list[str] = [] + cats.append(f"source:{src}") cats.append(f"severity:{severity}") if adv.package_ecosystem: cats.append(f"ecosystem:{adv.package_ecosystem}") @@ -97,6 +106,15 @@ def _build_item_xml(adv: NormalizedAdvisory) -> str: # also cve if present if adv.cve_id: cats.append(f"cve:{adv.cve_id}") + # extra cves for cert-tr + extra = getattr(adv, "extra_cves", []) or [] + for ec in extra[:5]: + cats.append(f"cve:{ec}") + # CWE identifier if present + for ident in getattr(adv, "identifiers", []) or []: + if isinstance(ident, dict) and ident.get("type") == "CWE": + cats.append(f"cwe:{ident.get('value')}") + break cat_xml = "\n".join(f" {escape_text(c)}" for c in cats) @@ -239,7 +257,12 @@ def _build_content_html(adv: NormalizedAdvisory) -> str: parts.append("") view_link = adv.html_url if _is_safe_url(adv.html_url) else "#" - parts.append(f'

View on GitHub

') + label = "View NVD entry" if getattr(adv, "source", "github") == "cert-tr" else "View on GitHub" + parts.append(f'

{label}

') + # source footer + src = getattr(adv, "source", "github") or "github" + if src == "cert-tr": + parts.append('

Source: CERT-TR / Siber Güvenlik Başkanlığı (via Proton Mail)

') html_str = "\n".join(parts) # Bound content size per item diff --git a/src/advisory_rss/server/app.py b/src/advisory_rss/server/app.py index bb6564a..202d5d6 100644 --- a/src/advisory_rss/server/app.py +++ b/src/advisory_rss/server/app.py @@ -1,4 +1,5 @@ # FastAPI app factory +# ruff: noqa: BLE001, S110 from __future__ import annotations @@ -112,10 +113,33 @@ def create_app( @app.get("/health") async def health() -> JSONResponse: meta = cache.get_cache_meta() + # per-source counts + try: + all_advs = cache.load_all() + github_count = sum(1 for a in all_advs if (getattr(a, "source", "github") or "github") == "github") + cert_tr_count = sum(1 for a in all_advs if getattr(a, "source", "") == "cert-tr") + except Exception: + github_count = None + cert_tr_count = None + # cert_tr diag + cert_tr_diag = None + try: + import json + + raw = cache.get_meta("cert_tr_diag") + if raw: + cert_tr_diag = json.loads(raw) + except Exception: + pass return JSONResponse( content={ "status": "ok" if not meta.rate_limited_until else "rate_limited", "advisories_count": meta.advisories_count, + "github_count": github_count, + "cert_tr_count": cert_tr_count, + "cert_tr_enabled": settings.enable_cert_tr, + "cert_tr_accounts": len(settings.get_proton_accounts()) if settings.enable_cert_tr else 0, + "cert_tr_diag": cert_tr_diag, "last_successful_sync": meta.last_successful_sync, "next_scheduled_sync": meta.next_scheduled_sync, "rate_limited_until": meta.rate_limited_until, @@ -134,26 +158,49 @@ async def health() -> JSONResponse: @app.get("/rss.xml") @app.get("/rss") async def rss(request: Request) -> Response: - if request.query_params: + # Optional source filter: ?source=all|github|cert-tr (default all = mixed) + src_filter = (request.query_params.get("source") or "all").lower().strip() + if src_filter not in ("all", "github", "cert-tr", "cert_tr"): + src_filter = "all" + if src_filter == "cert_tr": + src_filter = "cert-tr" + if request.query_params and "source" not in request.query_params: logger.debug( - "RSS query params ignored: %s path=%s", + "RSS query params ignored (no source): %s path=%s", _redact(str(request.query_params)), request.url.path, ) logger.debug( - "RSS request path=%s user_agent=%s", + "RSS request path=%s source=%s user_agent=%s", request.url.path, + src_filter, _redact(request.headers.get("user-agent", "")[:200]), ) - # Serve from cache only - never call GitHub here + # Serve from cache only - never call GitHub/IMAP here advisories = cache.load_sorted(limit=None) # load all sorted, builder caps by max_items + if src_filter != "all": + advisories = [a for a in advisories if (getattr(a, "source", "github") or "github") == src_filter] feed_user = cache.get_meta("authenticated_user") or "user" ttl_minutes = max(1, settings.refresh_interval // 60) + # Mixed feed title when both sources present + has_github = any((getattr(a, "source", "github") or "github") == "github" for a in advisories) + has_cert = any(getattr(a, "source", "") == "cert-tr" for a in advisories) + if has_github and has_cert and src_filter == "all": + feed_title = f"Security Advisories (GitHub + CERT-TR) - @{feed_user}" + feed_desc = f"Mixed feed: GitHub advisories by @{feed_user} + CERT-TR via Proton Mail" + elif src_filter == "cert-tr": + feed_title = "CERT-TR / Siber Güvenlik Başkanlığı Advisories" + feed_desc = "CERT-TR advisories via Proton Mail (siberguvenlik.gov.tr)" + else: + feed_title = f"GitHub Security Advisories - @{feed_user}" + feed_desc = f"Security advisories created by {feed_user} (author={feed_user}) via GitHub API - localhost-only feed" + # Keep feed_link sensible + feed_link = f"https://github.com/{feed_user}" if src_filter != "cert-tr" else "https://siberguvenlik.gov.tr" xml = build_rss( advisories, - feed_title=f"GitHub Security Advisories - @{feed_user}", - feed_link=f"https://github.com/{feed_user}", - feed_description=f"Security advisories created by {feed_user} (author={feed_user}) via GitHub API - localhost-only feed", + feed_title=feed_title, + feed_link=feed_link, + feed_description=feed_desc, max_items=settings.max_items, ttl_minutes=ttl_minutes, authenticated_user=feed_user, @@ -211,35 +258,68 @@ async def refresh(request: Request) -> JSONResponse: raise HTTPException(status_code=403, detail="Invalid refresh token") # Enforce streaming body size even for chunked (defense in depth) await _read_limited_body(request, MAX_POST_BYTES) + # Optional ?source= filter + q_source = (request.query_params.get("source") or "all").lower().strip() + if q_source not in ("all", "github", "cert-tr", "cert_tr"): + q_source = "all" + if q_source == "cert_tr": + q_source = "cert-tr" # Trigger sync background token = load_token(settings) - if not token: + # Refresh allows mixed: if no GitHub token but cert-tr enabled, still refresh cert-tr + need_github = q_source in ("all", "github") + if need_github and not token and settings.enable_cert_tr and q_source == "all": + need_github = False + elif need_github and not token: raise HTTPException(status_code=401, detail="GitHub token not configured") # Run sync inline but bounded try: - client = GitHubClient(settings, token, cache=cache) + all_advs = [] + diag_comb: dict[str, Any] = {"errors": []} + if need_github and token: + client = GitHubClient(settings, token, cache=cache) + try: + advs, diag = await client.sync_all(filter_mode=settings.filter_mode) + all_advs.extend(advs) + diag_comb["github"] = diag + diag_comb["errors"].extend(diag.get("errors") or []) + finally: + await client.close() + if q_source in ("all", "cert-tr") and settings.enable_cert_tr: + from advisory_rss.cert_tr.source import CertTrSource + + src = CertTrSource(settings) + advs_ct, diag_ct = src.fetch_all() + all_advs.extend(advs_ct) + diag_comb["cert_tr"] = diag_ct + diag_comb["errors"].extend(diag_ct.get("errors") or []) + if not all_advs: + raise HTTPException(status_code=502, detail=f"Refresh: no advisories (errors={diag_comb['errors'][:2]})") + count = cache.upsert_advisories(all_advs) + # keep user from GitHub if present + user = None + if diag_comb.get("github"): + user = diag_comb["github"].get("authenticated_login") + cache.mark_success(user=user) + nxt = (datetime.now(UTC) + timedelta(seconds=settings.refresh_interval)).isoformat() + cache.set_meta("next_scheduled_sync", nxt) try: - advs, diag = await client.sync_all(filter_mode=settings.filter_mode) - count = cache.upsert_advisories(advs) - cache.mark_success(user=diag.get("authenticated_login")) - # next sync - nxt = (datetime.now(UTC) + timedelta(seconds=settings.refresh_interval)).isoformat() - cache.set_meta("next_scheduled_sync", nxt) - logger.info( - "POST /refresh ok: count=%d login=%s", - count, - diag.get("authenticated_login"), - extra={"diag": diag}, - ) - return JSONResponse( - content={ - "status": "refreshed", - "count": count, - "diag": {k: v for k, v in diag.items() if k != "errors" or v}, - } - ) - finally: - await client.close() + import json + + if diag_comb.get("cert_tr"): + cache.set_meta("cert_tr_diag", json.dumps(diag_comb["cert_tr"], ensure_ascii=False)) + except Exception: + pass + logger.info("POST /refresh ok: count=%d", count, extra={"diag": diag_comb}) + return JSONResponse( + content={ + "status": "refreshed", + "count": count, + "diag": {k: v for k, v in diag_comb.items() if k != "errors" or v}, + } + ) + except HTTPException: + raise except (OSError, ValueError, RuntimeError) as e: msg = _redact(str(e)) logger.warning("Refresh failed: %s", msg, exc_info=True) @@ -266,10 +346,12 @@ async def unhandled(request: Request, exc: Exception): async def background_refresh_loop(settings: Settings, cache: CacheStore) -> None: # Run an immediate refresh on startup (after short grace) then periodic + # Mixed: GitHub + CERT-TR (IMAP is sync, wrap in thread) logger.info( - "background_refresh_loop starting interval=%ds filter=%s", + "background_refresh_loop starting interval=%ds filter=%s cert_tr=%s", settings.refresh_interval, settings.filter_mode, + settings.enable_cert_tr, ) first = True while True: @@ -280,25 +362,87 @@ async def background_refresh_loop(settings: Settings, cache: CacheStore) -> None await asyncio.sleep(2) first = False token = load_token(settings) - if not token: - logger.info("Background refresh skipped - no token") + # If neither source is usable, skip + if not token and not settings.enable_cert_tr: + logger.info("Background refresh skipped - no token and CERT-TR disabled") + continue + # Also skip if CERT-TR enabled but no Proton accounts and no token => nothing to do + if not token and settings.enable_cert_tr and not settings.get_proton_accounts(): + logger.info("Background refresh skipped - CERT-TR enabled but no Proton accounts and no GitHub token") continue try: - client = GitHubClient(settings, token, cache=cache) - try: - advs, diag = await client.sync_all(filter_mode=settings.filter_mode) - cache.upsert_advisories(advs) - cache.mark_success(user=diag.get("authenticated_login")) - nxt = (datetime.now(UTC) + timedelta(seconds=settings.refresh_interval)).isoformat() - cache.set_meta("next_scheduled_sync", nxt) + all_advs = [] + github_diag = None + cert_diag = None + errors: list[str] = [] + # GitHub (async) + if token: + client = GitHubClient(settings, token, cache=cache) + try: + advs, diag = await client.sync_all(filter_mode=settings.filter_mode) + all_advs.extend(advs) + github_diag = diag + errors.extend(diag.get("errors") or []) + except (OSError, ValueError, RuntimeError) as e: + msg = _redact(str(e)) + logger.warning("Background GitHub refresh failed: %s", msg, exc_info=True) + errors.append(msg) + # keep stale, don't abort cert-tr + finally: + await client.close() + # CERT-TR (sync IMAP -> run in thread) + if settings.enable_cert_tr: + try: + from advisory_rss.cert_tr.source import CertTrSource + + def _fetch_ct(): + src = CertTrSource(settings) + return src.fetch_all() + + advs_ct, diag_ct = await asyncio.to_thread(_fetch_ct) + cert_diag = diag_ct + errors.extend(diag_ct.get("errors") or []) + if advs_ct: + all_advs.extend(advs_ct) + except (OSError, ValueError, RuntimeError) as e: + msg = _redact(str(e)) + logger.warning("Background CERT-TR refresh failed: %s", msg, exc_info=True) + errors.append(msg) + if not all_advs and errors: + # Report but keep stale + msg = "; ".join(errors[:3]) + cache.mark_error(msg) + logger.warning("Background refresh: no advisories, errors=%s", msg) + elif all_advs: + # Dedup already done per source; final merge dedup by ghsa_id + cache.upsert_advisories(all_advs) + user = (github_diag or {}).get("authenticated_login") if github_diag else None + cache.mark_success(user=user or cache.get_meta("authenticated_user")) + try: + import json + + if cert_diag is not None: + cache.set_meta("cert_tr_diag", json.dumps(cert_diag, ensure_ascii=False)) + except Exception: + pass + gh_count = 0 + if isinstance(github_diag, dict): + try: + gh_count = int(github_diag.get("filtered_count") or 0) + except Exception: + gh_count = 0 + ct_count = int(cert_diag.get("parsed") or 0) if isinstance(cert_diag, dict) else 0 logger.info( - "Background refresh OK: %d advisories login=%s", - len(advs), - diag.get("authenticated_login"), - extra={"diag": diag}, + "Background refresh OK: total=%d github=%d cert_tr=%d errors=%d", + len(all_advs), + gh_count, + ct_count, + len(errors), ) - finally: - await client.close() + else: + logger.info("Background refresh: no new advisories") + nxt = (datetime.now(UTC) + timedelta(seconds=settings.refresh_interval)).isoformat() + cache.set_meta("next_scheduled_sync", nxt) except (OSError, ValueError, RuntimeError) as e: msg = _redact(str(e)) logger.warning("Background refresh failed, keeping stale cache: %s", msg, exc_info=True) diff --git a/tests/unit/test_cert_tr_imap.py b/tests/unit/test_cert_tr_imap.py new file mode 100644 index 0000000..adc6781 --- /dev/null +++ b/tests/unit/test_cert_tr_imap.py @@ -0,0 +1,60 @@ +from unittest.mock import MagicMock, patch + +from advisory_rss.cert_tr.imap import fetch_raw_emails + +SAMPLE_RAW = b"From: cve@siberguvenlik.gov.tr\r\nSubject: Test CVE-2026-0001\r\n\r\nBody CVE-2026-0001" + +def _mock_imap(data_bytes_list): + """Create mock IMAP object that returns uids and fetches.""" + mock = MagicMock() + # select + mock.select.return_value = ("OK", [b"1"]) + # search returns uids + mock.search.return_value = ("OK", [b"1 2"]) + # fetch returns tuple per uid + def fetch_side(uid, _what): + # uid can be bytes like b"2" or b"1" + if isinstance(uid, bytes): + uid_s = uid.decode() + else: + uid_s = str(uid) + # map uid to sample + idx = 0 if uid_s == "2" else 1 + raw = data_bytes_list[idx] if idx < len(data_bytes_list) else data_bytes_list[0] + return ("OK", [(b"1 (BODY[] {10}", raw)]) + mock.fetch.side_effect = fetch_side + return mock + +def test_fetch_never_marks_read(): + acc = {"email": "a@proton.me", "password": "p", "folder": "INBOX", "host": "127.0.0.1", "port": "1143", "security": "STARTTLS"} + with patch("advisory_rss.cert_tr.imap.connect_imap") as mock_connect: + mock = _mock_imap([SAMPLE_RAW, SAMPLE_RAW]) + mock_connect.return_value = mock + raws = fetch_raw_emails(acc, ["siberguvenlik.gov.tr"], max_mails=2) + # Should have used BODY.PEEK[] + assert mock.fetch.called + for call_args in mock.fetch.call_args_list: + args, _kwargs = call_args + assert "BODY.PEEK" in args[1] + # Should have selected readonly=True + assert mock.select.called + for c in mock.select.call_args_list: + args, kwargs = c + assert kwargs.get("readonly") is True + assert len(raws) == 2 + +def test_fetch_multi_account_distinct_folders(): + from advisory_rss.config.settings import Settings + s = Settings(_env_file="/dev/null", ENABLE_CERT_TR="true", PROTON_BRIDGE_EMAILS="a@proton.me,b@proton.me", PROTON_BRIDGE_PASSWORDS="p1,p2", PROTON_IMAP_FOLDERS="INBOX,INBOX.CERT-TR", _env_file_encoding="utf-8") + accs = s.get_proton_accounts() + assert accs[0]["folder"] == "INBOX" + assert accs[1]["folder"] == "INBOX.CERT-TR" + # ensure each fetch would select its folder + with patch("advisory_rss.cert_tr.imap.connect_imap") as mock_connect: + m1 = _mock_imap([SAMPLE_RAW]) + m2 = _mock_imap([SAMPLE_RAW]) + mock_connect.side_effect = [m1, m2] + r1 = fetch_raw_emails(accs[0], ["siberguvenlik.gov.tr"], max_mails=1) + r2 = fetch_raw_emails(accs[1], ["siberguvenlik.gov.tr"], max_mails=1) + assert m1.select.call_args[0][0] == "INBOX" + assert m2.select.call_args[0][0] == "INBOX.CERT-TR" diff --git a/tests/unit/test_cert_tr_parser.py b/tests/unit/test_cert_tr_parser.py new file mode 100644 index 0000000..6cdd1d3 --- /dev/null +++ b/tests/unit/test_cert_tr_parser.py @@ -0,0 +1,53 @@ +from pathlib import Path + +from advisory_rss.cert_tr.parser import is_cert_tr_sender, parse_cert_tr_email + + +def test_cert_tr_parse_sample(): + raw = Path("tests/fixtures/cert_tr/CVE-2026-92083.eml").read_bytes() + adv = parse_cert_tr_email(raw) + assert adv is not None + assert adv.ghsa_id == "CERT-TR-CVE-2026-92083" + assert adv.cve_id == "CVE-2026-92083" + assert adv.source == "cert-tr" + assert adv.package_name == "pardus-etap-settings" + assert any(i["value"] == "CWE-862" for i in adv.identifiers) + assert "pardus-etap-settings" in adv.summary + assert adv.author_login == "cve@siberguvenlik.gov.tr" + assert adv.publisher_login == "siberguvenlik.gov.tr" + assert adv.repository_full_name == "cert-tr/pardus-etap-settings" + assert "https://nvd.nist.gov/vuln/detail/CVE-2026-92083" in adv.references + + +def test_cert_tr_sender_allowlist(): + assert is_cert_tr_sender("Ürün Güvenliği Koordinasyon Ekibi ", ["siberguvenlik.gov.tr"]) + assert is_cert_tr_sender("cve@siberguvenlik.gov.tr", ["siberguvenlik.gov.tr"]) + assert not is_cert_tr_sender("noreply@github.com", ["siberguvenlik.gov.tr"]) + assert is_cert_tr_sender("test@usom.gov.tr", ["siberguvenlik.gov.tr", "usom.gov.tr"]) + + +def test_cert_tr_parser_second_mail_update(): + raw1 = Path("tests/fixtures/cert_tr/CVE-2026-92083.eml").read_bytes() + adv1 = parse_cert_tr_email(raw1) + assert adv1 is not None + # Simulate update: same CVE but different body (e.g., patched) + raw2 = raw1.replace(b"CWE-862", b"CWE-79").replace(b"13:23:43 +0000", b"15:23:43 +0000") + adv2 = parse_cert_tr_email(raw2) + assert adv2 is not None + assert adv1.ghsa_id == adv2.ghsa_id + # updated time should differ? Date changed + assert adv2.identifiers != adv1.identifiers or adv2.updated_at != adv1.updated_at + + +def test_cert_tr_parser_handles_quoted_printable_charset(): + raw = Path("tests/fixtures/cert_tr/CVE-2026-92083.eml").read_bytes() + adv = parse_cert_tr_email(raw) + assert adv is not None + # Check Turkish chars decoded correctly + assert "TÜBİTAK" in adv.description or "TUBITAK" in adv.description or "BİLGEM" in adv.description + + +def test_cert_tr_parser_no_cve_skipped(): + raw = b"From: test@siberguvenlik.gov.tr\r\nSubject: Hello\r\nDate: Tue, 15 Sep 2026 13:23:43 +0000\r\nMessage-Id: \r\nContent-Type: text/plain; charset=utf-8\r\n\r\nJust a hello, no CVE here." + adv = parse_cert_tr_email(raw) + assert adv is None diff --git a/tests/unit/test_cert_tr_rss.py b/tests/unit/test_cert_tr_rss.py new file mode 100644 index 0000000..bc6031e --- /dev/null +++ b/tests/unit/test_cert_tr_rss.py @@ -0,0 +1,47 @@ +from datetime import UTC, datetime + +from advisory_rss.cache.models import NormalizedAdvisory +from advisory_rss.rss.builder import build_rss + + +def test_mixed_rss(): + gh = NormalizedAdvisory( + ghsa_id="GHSA-xxxx-yyyy-zzzz", + summary="GH vuln", + description="gh", + severity="high", + html_url="https://github.com/advisories/GHSA-xxxx", + author_login="user", + published_at=datetime(2026, 9, 20, tzinfo=UTC), + updated_at=datetime(2026, 9, 20, tzinfo=UTC), + source="github", + ) + ct = NormalizedAdvisory( + ghsa_id="CERT-TR-CVE-2026-92083", + cve_id="CVE-2026-92083", + summary="pardus-etap-settings: CVE-2026-92083 (CWE-862)", + description="cert", + html_url="https://nvd.nist.gov/vuln/detail/CVE-2026-92083", + author_login="cve@siberguvenlik.gov.tr", + published_at=datetime(2026, 9, 15, tzinfo=UTC), + updated_at=datetime(2026, 9, 15, tzinfo=UTC), + source="cert-tr", + package_name="pardus-etap-settings", + identifiers=[{"type": "CWE", "value": "CWE-862"}], + ) + xml = build_rss([gh, ct], feed_title="T", feed_link="https://example.com", feed_description="d", max_items=100, ttl_minutes=10, authenticated_user="user") + assert "source:github" in xml + assert "source:cert-tr" in xml + assert "[CERT-TR]" in xml + assert "cwe:CWE-862" in xml + # order: GHSA newer first if sorted inside builder + assert xml.index("GHSA-xxxx") < xml.index("CERT-TR-CVE") + +def test_cert_tr_rss_filter(): + gh = NormalizedAdvisory(ghsa_id="GHSA-1", summary="g", source="github", updated_at=datetime(2026,9,20,tzinfo=UTC), published_at=datetime(2026,9,20,tzinfo=UTC)) + ct = NormalizedAdvisory(ghsa_id="CERT-TR-CVE-1", summary="c", source="cert-tr", updated_at=datetime(2026,9,15,tzinfo=UTC), published_at=datetime(2026,9,15,tzinfo=UTC)) + # simulate server filter + all_adv = [gh, ct] + filtered = [a for a in all_adv if getattr(a,"source","github")=="cert-tr"] + assert len(filtered)==1 + assert filtered[0].ghsa_id=="CERT-TR-CVE-1" diff --git a/tests/unit/test_cert_tr_settings.py b/tests/unit/test_cert_tr_settings.py new file mode 100644 index 0000000..645c8fe --- /dev/null +++ b/tests/unit/test_cert_tr_settings.py @@ -0,0 +1,52 @@ +from advisory_rss.config.settings import Settings + + +def test_single_account(): + s = Settings(_env_file="/dev/null", ENABLE_CERT_TR="true", PROTON_BRIDGE_EMAIL="a@proton.me", PROTON_BRIDGE_PASSWORD="p1", PROTON_IMAP_FOLDER="INBOX", _env_file_encoding="utf-8") + accs = s.get_proton_accounts() + assert len(accs) == 1 + assert accs[0]["email"] == "a@proton.me" + assert accs[0]["folder"] == "INBOX" + assert accs[0]["password"] == "p1" + + +def test_multi_account_emails_folders(): + s = Settings( + _env_file="/dev/null", + ENABLE_CERT_TR="true", + PROTON_BRIDGE_EMAILS="a@proton.me, b@proton.me", + PROTON_BRIDGE_PASSWORDS="p1,p2", + PROTON_IMAP_FOLDERS="INBOX,INBOX.CERT-TR", + _env_file_encoding="utf-8", + ) + accs = s.get_proton_accounts() + assert len(accs) == 2 + assert accs[0]["folder"] == "INBOX" + assert accs[1]["folder"] == "INBOX.CERT-TR" + assert accs[1]["password"] == "p2" + + +def test_multi_account_fallback_single_password(): + s = Settings( + _env_file="/dev/null", + ENABLE_CERT_TR="true", + PROTON_BRIDGE_EMAILS="a@proton.me, b@proton.me", + PROTON_BRIDGE_PASSWORD="single", + PROTON_IMAP_FOLDER="INBOX", + _env_file_encoding="utf-8", + ) + accs = s.get_proton_accounts() + assert len(accs) == 2 + assert accs[0]["password"] == "single" + assert accs[1]["password"] == "single" + + +def test_folder_validation(): + s = Settings(_env_file="/dev/null", PROTON_IMAP_FOLDER="INBOX.CERT-TR", _env_file_encoding="utf-8") + assert s.proton_imap_folder == "INBOX.CERT-TR" + + +def test_sender_allowlist(): + s = Settings(_env_file="/dev/null", CERT_TR_SENDER_ALLOWLIST="siberguvenlik.gov.tr, usom.gov.tr", _env_file_encoding="utf-8") + assert "siberguvenlik.gov.tr" in s.cert_tr_sender_list() + assert "usom.gov.tr" in s.cert_tr_sender_list() From 849240d3a877e9f32703f1455ea36c15d3b9e22f Mon Sep 17 00:00:00 2001 From: Lunixizm0 <216941080+Lunixizm0@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:29:05 +0300 Subject: [PATCH 2/4] feat: Enhance CERT-TR support for Gmail alongside Proton Bridge --- .env.example | 18 ++- scripts/fixtures_generator.py | 2 +- src/advisory_rss/cache/store.py | 23 ++-- src/advisory_rss/cert_tr/imap.py | 102 ++++++++-------- src/advisory_rss/cert_tr/parser.py | 62 ++++------ src/advisory_rss/cert_tr/source.py | 24 ++-- src/advisory_rss/cli.py | 25 ++-- src/advisory_rss/config/constants.py | 6 +- src/advisory_rss/config/settings.py | 171 ++++++++++++++++++++++++++- src/advisory_rss/logging_config.py | 13 +- src/advisory_rss/server/app.py | 28 ++--- tests/unit/test_cert_tr_imap.py | 47 +++++++- tests/unit/test_cert_tr_settings.py | 54 +++++++++ 13 files changed, 424 insertions(+), 151 deletions(-) diff --git a/.env.example b/.env.example index 4753641..f46cd26 100644 --- a/.env.example +++ b/.env.example @@ -51,8 +51,9 @@ CACHE_PATH=cache/advisories.db # MAX_REPOS=1000 # MAX_PAGES_PER_REPO=100 -# CERT-TR / Siber Güvenlik Başkanlığı via Proton Mail Bridge (localhost IMAP) +# CERT-TR / Siber Güvenlik Başkanlığı via Proton Bridge or Gmail IMAP (never marks as read) # When false (default) only GitHub source is active. RSS is mixed (GitHub + CERT-TR) when enabled. +# Works with both Proton Bridge (127.0.0.1) and Gmail (imap.gmail.com) simultaneously. # ENABLE_CERT_TR=false # For single account: # PROTON_BRIDGE_EMAIL=mail@proton.me @@ -66,7 +67,20 @@ CACHE_PATH=cache/advisories.db # PROTON_BRIDGE_HOST=127.0.0.1 # PROTON_IMAP_PORT=1143 # PROTON_IMAP_SECURITY=STARTTLS # STARTTLS | SSL | NONE (Bridge uses STARTTLS, self-signed) -# Filtering & limits: +# Gmail alternative (same CERT-TR mails via Gmail IMAP, also never marks as read) +# GMAIL_EMAIL=your@gmail.com +# GMAIL_APP_PASSWORD=abcd efgh ijkl mnop # Gmail App Password (16 chars, spaces preserved; for multi use comma) +# GMAIL_IMAP_FOLDER=INBOX +# For multiple Gmail accounts (each mail + inbox separately): +# GMAIL_EMAILS=your@gmail.com,second@gmail.com +# GMAIL_APP_PASSWORDS=app_pass1,"app pass 2 with spaces" # comma-separated, spaces inside preserved +# GMAIL_IMAP_FOLDERS=INBOX,INBOX +# Gmail IMAP (usually don't change): +# GMAIL_IMAP_HOST=imap.gmail.com +# GMAIL_IMAP_PORT=993 +# GMAIL_IMAP_SECURITY=SSL # SSL for Gmail, STARTTLS for Proton Bridge + +# Filtering & limits (applies to both Proton and Gmail): # CERT_TR_SENDER_ALLOWLIST=siberguvenlik.gov.tr # CERT_TR_MAX_MAILS=200 # CERT_TR_SEARCH_DAYS= # empty = all time, or e.g. 30 for last 30 days diff --git a/scripts/fixtures_generator.py b/scripts/fixtures_generator.py index be690d4..a02d217 100755 --- a/scripts/fixtures_generator.py +++ b/scripts/fixtures_generator.py @@ -308,7 +308,7 @@ async def main_async(args: argparse.Namespace) -> int: p.unlink(missing_ok=True) return 0 except (httpx.HTTPError, OSError, ValueError, RuntimeError) as e: - logger.error("fixtures_generator failed: %s", _redact(str(e)), exc_info=True) + logger.exception("fixtures_generator failed") print(f"ERROR: {_redact(str(e))}", file=sys.stderr) import traceback diff --git a/src/advisory_rss/cache/store.py b/src/advisory_rss/cache/store.py index 20eb983..3bf6520 100644 --- a/src/advisory_rss/cache/store.py +++ b/src/advisory_rss/cache/store.py @@ -90,11 +90,18 @@ def _init_schema(self) -> None: logger.debug("migration backfill skipped %s: %s", ghsa_id, ee) except sqlite3.Error as ee: logger.debug("backfill query failed: %s", ee) + # Cleanup legacy false positives: non-CVE CERT-TR entries (hash-based IDs) + try: + cur = self._conn.execute("DELETE FROM advisories WHERE ghsa_id LIKE 'CERT-TR-MSG-%'") + if cur.rowcount and cur.rowcount > 0: + logger.info("Cleaned up %d legacy CERT-TR non-CVE advisories", cur.rowcount) + except sqlite3.Error as ce: + logger.debug("Cleanup legacy advisories failed: %s", ce) except sqlite3.Error as me: logger.warning("Migration check failed: %s", me) logger.debug("cache schema ready db=%s", self.db_path) - except sqlite3.Error as e: - logger.error("Cache schema init failed: %s", e, exc_info=True) + except sqlite3.Error: + logger.exception("Cache schema init failed") def upsert_advisories(self, advisories: list[NormalizedAdvisory]) -> int: """Upsert list; returns count. Uses parameterized queries.""" @@ -141,8 +148,8 @@ def upsert_advisories(self, advisories: list[NormalizedAdvisory]) -> int: logger.warning("Failed to upsert %s: %s", adv.ghsa_id, e) cur.execute("COMMIT") logger.info("upsert_advisories committed count=%d", count) - except sqlite3.Error as e: - logger.error("Cache upsert transaction failed: %s", e, exc_info=True) + except sqlite3.Error: + logger.exception("Cache upsert transaction failed") try: self._conn.execute("ROLLBACK") except sqlite3.Error: @@ -163,8 +170,8 @@ def load_all(self) -> list[NormalizedAdvisory]: logger.warning("Corrupt cache row skipped: %s", e) logger.debug("load_all fetched %d rows -> %d advisories", len(rows), len(out)) return out - except sqlite3.Error as e: - logger.error("Cache load failed: %s", e, exc_info=True) + except sqlite3.Error: + logger.exception("Cache load failed") return [] def load_sorted(self, limit: int | None = None) -> list[NormalizedAdvisory]: @@ -199,8 +206,8 @@ def clear(self) -> None: self._conn.execute("DELETE FROM etags") self._conn.execute("COMMIT") logger.info("cache cleared") - except sqlite3.Error as e: - logger.error("Cache clear failed: %s", e, exc_info=True) + except sqlite3.Error: + logger.exception("Cache clear failed") try: self._conn.execute("ROLLBACK") except sqlite3.Error: diff --git a/src/advisory_rss/cert_tr/imap.py b/src/advisory_rss/cert_tr/imap.py index 7f9916a..12a0578 100644 --- a/src/advisory_rss/cert_tr/imap.py +++ b/src/advisory_rss/cert_tr/imap.py @@ -1,5 +1,4 @@ -"""Proton Bridge IMAP fetcher - localhost only, never marks read.""" -# ruff: noqa: BLE001, DTZ003, S110, RUF059 +"""IMAP fetcher for CERT-TR mails via Proton Bridge and Gmail (never marks read).""" from __future__ import annotations @@ -7,17 +6,33 @@ import logging import ssl import time -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta logger = logging.getLogger(__name__) +# Allowed IMAP hosts: Proton Bridge loopback + Gmail +ALLOWED_HOSTS = { + "127.0.0.1", + "::1", + "localhost", + "imap.gmail.com", + "imap.googlemail.com", +} + # Never mark read: we always use BODY.PEEK and SELECT readonly=True -def _ssl_context() -> ssl.SSLContext: +def _ssl_context_for_host(host: str) -> ssl.SSLContext: + # Proton Bridge uses self-signed cert -> no verification + if host in {"127.0.0.1", "::1", "localhost"}: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx + # Gmail and others -> verify ctx = ssl.create_default_context() - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED return ctx @@ -28,24 +43,21 @@ def connect_imap(account: dict[str, str]) -> imaplib.IMAP4: email = account.get("email", "") password = account.get("password", "") - # Loopback guard - only allow loopback - if host not in ("127.0.0.1", "::1", "localhost"): - raise ValueError(f"Proton IMAP host must be loopback, got {host!r}") + if host not in ALLOWED_HOSTS: + raise ValueError(f"IMAP host must be one of {sorted(ALLOWED_HOSTS)}, got {host!r}") - ctx = _ssl_context() + ctx = _ssl_context_for_host(host) if security == "SSL": mail = imaplib.IMAP4_SSL(host, port, ssl_context=ctx) elif security == "STARTTLS": mail = imaplib.IMAP4(host, port) try: mail.starttls(ssl_context=ctx) - except Exception as e: + except (OSError, imaplib.IMAP4.error, ssl.SSLError) as e: logger.warning("STARTTLS failed for %s:%s - trying without TLS: %s", host, port, e) - # already connected without TLS; still try login else: # NONE mail = imaplib.IMAP4(host, port) - # Login - Bridge password is not Proton account password try: mail.login(email, password) except imaplib.IMAP4.error as e: @@ -57,16 +69,11 @@ def connect_imap(account: dict[str, str]) -> imaplib.IMAP4: def _build_search_criteria(allowlist: list[str], search_days: int | None) -> list[str]: parts: list[str] = [] - # FROM filter - build OR chain if multiple if allowlist: - # Use first allowlist entry for server-side filter; rest filtered client-side to keep simple - # Or build OR for 2 entries max? We'll use first dominant domain domain = allowlist[0] - # IMAP FROM search is substring match; domain is enough parts.append(f'FROM "{domain}"') if search_days is not None and search_days >= 0: - # SINCE date is inclusive - since_date = (datetime.utcnow() - timedelta(days=search_days)).strftime("%d-%b-%Y") + since_date = (datetime.now(UTC) - timedelta(days=search_days)).strftime("%d-%b-%Y") parts.append(f'SINCE "{since_date}"') if not parts: parts.append("ALL") @@ -79,7 +86,7 @@ def fetch_raw_emails( max_mails: int = 200, search_days: int | None = None, ) -> list[bytes]: - """Fetch raw RFC822 bytes for one Proton account, never marking as read. + """Fetch raw RFC822 bytes for one account, never marking as read. Returns list of raw bytes, newest first (reverse UID). Handles reconnect every 25 to avoid Bridge drops. @@ -91,14 +98,12 @@ def fetch_raw_emails( mail = None try: mail = connect_imap(account) - # Select readonly to guarantee no \Seen flag can be set even on bug typ: str = "" data: list[bytes] = [] try: typ, data = mail.select(folder, readonly=True) # type: ignore[assignment] - except imaplib.IMAP4.error as e: # noqa: BLE001 + except imaplib.IMAP4.error as e: logger.warning("IMAP select folder %r failed for %s: %s", folder, email, e) - # try INBOX fallback if folder.upper() != "INBOX": try: typ2, _data2 = mail.select("INBOX", readonly=True) # type: ignore[assignment] @@ -108,7 +113,8 @@ def fetch_raw_emails( typ, data = typ2, _data2 # type: ignore[assignment] else: return [] - except Exception: # noqa: BLE001 + except (OSError, imaplib.IMAP4.error) as fallback_e: + logger.debug("Fallback select failed for %s: %s", email, fallback_e) return [] else: return [] @@ -116,15 +122,11 @@ def fetch_raw_emails( logger.warning("IMAP select %r returned %s for %s: %s", folder, typ, email, data) return [] - # Build search criteria = _build_search_criteria(allowlist, search_days) - # IMAP search: join with space (AND) search_query = " ".join(criteria) if len(criteria) == 1 else "(" + " ".join(criteria) + ")" - # Actually need to handle OR case; simplified to single FROM if len(allowlist) > 1: - # Client-side filter fallback: search with SINCE only or ALL, then filter if search_days is not None: - search_query = f'SINCE "{(datetime.utcnow() - timedelta(days=search_days)).strftime("%d-%b-%Y")}"' + search_query = f'SINCE "{(datetime.now(UTC) - timedelta(days=search_days)).strftime("%d-%b-%Y")}"' else: search_query = "ALL" @@ -140,33 +142,24 @@ def fetch_raw_emails( uids = data[0].split() if not uids: return [] - # Newest first: reverse UID list (UIDs increase) uids = list(reversed(uids)) - # Respect max_mails (per account) if len(uids) > max_mails: uids = uids[:max_mails] - # Client-side allowlist filtering if we used ALL fallback - # We'll still fetch and filter by From header later in parser, but to avoid fetching too many, - # we can peek headers first for From? Simpler: fetch all and let parser filter. - # For >1 allowlist and fallback ALL, we will filter after fetch via From header check. - - # Batch fetch with reconnect every 25 batch_size = 25 fetched = 0 for i in range(0, len(uids), batch_size): batch = uids[i : i + batch_size] - # reconnect every batch after first to avoid Bridge drops if i > 0: try: mail.logout() - except Exception: - pass + except (OSError, imaplib.IMAP4.error) as e: + logger.debug("Logout during reconnect failed for %s: %s", email, e) time.sleep(0.3) try: mail = connect_imap(account) mail.select(folder, readonly=True) - except Exception as e: + except (OSError, imaplib.IMAP4.error, ssl.SSLError, ValueError) as e: logger.warning("IMAP reconnect failed %s batch %d: %s", email, i, e) break @@ -176,7 +169,6 @@ def fetch_raw_emails( typ, fdata = mail.fetch(uid_str, "(BODY.PEEK[])") if typ != "OK" or not fdata: continue - # fdata is list of tuples; extract raw bytes for item in fdata: if isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], bytes): raw_list.append(item[1]) @@ -185,24 +177,36 @@ def fetch_raw_emails( raw_list.append(item) fetched += 1 except imaplib.IMAP4.error as e: - logger.debug("IMAP fetch uid %s failed %s: %s", uid.decode(errors="ignore"), email, e) + try: + uid_s = uid.decode(errors="ignore") if isinstance(uid, bytes) else str(uid) + except (ValueError, UnicodeDecodeError) as de: + uid_s = str(uid) + logger.debug("UID decode failed: %s", de) + logger.debug("IMAP fetch uid %s failed %s: %s", uid_s, email, e) continue - except Exception as e: + except (OSError, ValueError, RuntimeError) as e: logger.warning("IMAP fetch unexpected %s: %s", email, e) continue - logger.info("IMAP fetch done %s/%r: requested %d, fetched %d raw", email, folder, len(uids), len(raw_list)) + logger.info( + "IMAP fetch done %s/%r: requested %d, fetched %d raw", + email, + folder, + len(uids), + len(raw_list), + ) return raw_list - except Exception as e: + except (OSError, imaplib.IMAP4.error, ssl.SSLError, ValueError, RuntimeError) as e: logger.warning("IMAP fetch_raw_emails failed %s/%r: %s", email, folder, e, exc_info=True) return [] finally: if mail is not None: try: mail.logout() - except Exception: + except (OSError, imaplib.IMAP4.error) as e: + logger.debug("Final logout failed for %s: %s", email, e) try: mail.close() - except Exception: - pass + except (OSError, imaplib.IMAP4.error) as close_e: + logger.debug("Final close failed for %s: %s", email, close_e) diff --git a/src/advisory_rss/cert_tr/parser.py b/src/advisory_rss/cert_tr/parser.py index f937463..566ba1d 100644 --- a/src/advisory_rss/cert_tr/parser.py +++ b/src/advisory_rss/cert_tr/parser.py @@ -7,11 +7,9 @@ - CVE-dedup ID: CERT-TR-CVE-YYYY-NNNNN or fallback to Message-Id hash - Never marks as read (parser only) """ -# ruff: noqa: BLE001, S112, SIM102 from __future__ import annotations -import hashlib import html import logging import re @@ -52,7 +50,7 @@ def decode_rfc2047(s: str | None) -> str: else: out += raw return out - except Exception as e: + except (LookupError, ValueError, UnicodeDecodeError, AttributeError) as e: logger.debug("decode_rfc2047 failed %r: %s", s[:80], e) return str(s) @@ -75,7 +73,7 @@ def _parse_date(date_str: str | None) -> datetime | None: if dt and dt.tzinfo is None: dt = dt.replace(tzinfo=UTC) return dt - except Exception as e: + except (ValueError, TypeError, AttributeError) as e: logger.debug("date parse failed %r: %s", date_str, e) return None @@ -132,7 +130,7 @@ def _extract_body(msg: EmailMessage) -> tuple[str, str | None]: break except LookupError: continue - except Exception: # noqa: BLE001 + except (ValueError, UnicodeDecodeError): continue if text is None: text = payload.decode("utf-8", errors="replace") # type: ignore[union-attr] @@ -244,15 +242,15 @@ def _build_summary(subject_decoded: str, product: str | None, cve: str | None, c def parse_cert_tr_email(raw_bytes: bytes) -> NormalizedAdvisory | None: - """Parse raw RFC822 bytes from Proton Bridge into NormalizedAdvisory. + """Parse raw RFC822 bytes from Proton/Bridge or Gmail into NormalizedAdvisory. - Returns None if sender not in allowlist or no CVE and not enough data. - Uses CVE-based id for dedup (CERT-TR-CVE-...), else Message-Id hash. + Returns None if no CVE is assigned (strict filter to avoid false positives). + Uses CVE-based id for dedup (CERT-TR-CVE-...). Never marks mail read. """ try: msg = BytesParser(policy=policy.default).parsebytes(raw_bytes) - except Exception as e: + except (ValueError, TypeError, AttributeError) as e: logger.warning("Failed to parse email bytes: %s", e) return None @@ -283,32 +281,23 @@ def parse_cert_tr_email(raw_bytes: bytes) -> NormalizedAdvisory | None: # If no CVE found, still maybe useful? Require at least product or cwe or cve # For strict CERT-TR, require CVE; otherwise skip to avoid noise from unrelated mail + if not cve_primary and message_id: + m = CVE_RE.search(message_id) + if m: + cve_primary = m.group(0).upper() + if cve_primary not in cves: + cves.append(cve_primary) + if not cve_primary: - # also check message_id contains CVE? - if message_id: - m = CVE_RE.search(message_id) - if m: - cve_primary = m.group(0).upper() - if cve_primary not in cves: - cves.append(cve_primary) - - if not cve_primary and not product and not cwe: - logger.debug("Skipping non-CERT-TR mail subject=%r from=%r", subject[:60], from_decoded[:60]) + logger.debug( + "Skipping CERT-TR mail without CVE subject=%r from=%r", + subject[:60], + from_decoded[:60], + ) return None - # If still no CVE, we cannot make stable id from CVE; use message-id hash but mark as cert-tr generic - # For now require CVE to produce advisory; if no CVE, create id from hash but keep cve_id None - # We will allow it but id will be hash-based - # Validation: sender domain check is done at source level, not here - # ID generation: CERT-TR-CVE-xxx priority - if cve_primary: - ghsa_id = f"CERT-TR-{cve_primary.upper()}" - else: - # hash message-id + date + subject - h = hashlib.sha256() - h.update((message_id or subject or plain_text[:100]).encode("utf-8", errors="ignore")) - digest = h.hexdigest()[:12].upper() - ghsa_id = f"CERT-TR-MSG-{digest}" + # ID generation: strictly CVE-based (prevents false positives) + ghsa_id = f"CERT-TR-{cve_primary.upper()}" # Extra CVEs beyond primary extra_cves = [c for c in cves if c != cve_primary] @@ -332,13 +321,8 @@ def parse_cert_tr_email(raw_bytes: bytes) -> NormalizedAdvisory | None: # also include siberguvenlik portal (generic) refs.append("https://siberguvenlik.gov.tr") - # Links: html_url fallback to siberguvenlik - html_url = "" - if cve_primary: - # placeholder deep link not known; use NVD - html_url = f"https://nvd.nist.gov/vuln/detail/{cve_primary}" - else: - html_url = "https://siberguvenlik.gov.tr" + # Links: CVE-based NVD link (strict CVE-only) + html_url = f"https://nvd.nist.gov/vuln/detail/{cve_primary}" # Severity: try to infer from text (kritik/yüksek/orta/düşük) severity = None diff --git a/src/advisory_rss/cert_tr/source.py b/src/advisory_rss/cert_tr/source.py index 5ecd174..d09f7e9 100644 --- a/src/advisory_rss/cert_tr/source.py +++ b/src/advisory_rss/cert_tr/source.py @@ -1,8 +1,8 @@ """CERT-TR source orchestrator: IMAP fetch -> parser -> NormalizedAdvisory list.""" -# ruff: noqa: BLE001 from __future__ import annotations +import imaplib import logging from typing import Any @@ -35,9 +35,9 @@ def fetch_all(self) -> tuple[list[NormalizedAdvisory], dict[str, Any]]: logger.debug("CERT-TR disabled (ENABLE_CERT_TR=false)") return [], diag - accounts = self.settings.get_proton_accounts() + accounts = self.settings.get_cert_tr_accounts() if not accounts: - msg = "ENABLE_CERT_TR=true but no PROTON_BRIDGE_EMAIL/EMAILS configured" + msg = "ENABLE_CERT_TR=true but no PROTON_BRIDGE_EMAIL/EMAILS or GMAIL_EMAIL/EMAILS configured" logger.warning(msg) diag["errors"].append(msg) return [], diag @@ -48,24 +48,25 @@ def fetch_all(self) -> tuple[list[NormalizedAdvisory], dict[str, Any]]: all_advs: list[NormalizedAdvisory] = [] per_account_max = max_mails - # If multiple accounts, distribute budget if len(accounts) > 1: per_account_max = max(20, max_mails // len(accounts)) for acc in accounts: email = acc.get("email", "unknown") folder = acc.get("folder", "INBOX") - # Validate credentials + provider = acc.get("provider", "unknown") if not acc.get("password"): - msg = f"Proton account {email}/{folder}: missing PROTON_BRIDGE_PASSWORD - skipping" + msg = f"{provider} account {email}/{folder}: missing password/app-password - skipping" logger.warning(msg) diag["errors"].append(msg) continue diag["accounts"] += 1 try: - raw_list = cert_imap.fetch_raw_emails(acc, allowlist, max_mails=per_account_max, search_days=search_days) - except Exception as e: - msg = f"IMAP error {email}/{folder}: {e}" + raw_list = cert_imap.fetch_raw_emails( + acc, allowlist, max_mails=per_account_max, search_days=search_days + ) + except (OSError, ValueError, RuntimeError, imaplib.IMAP4.error) as e: + msg = f"IMAP error {email}/{folder} ({provider}): {e}" logger.warning(msg) diag["errors"].append(msg) continue @@ -76,16 +77,13 @@ def fetch_all(self) -> tuple[list[NormalizedAdvisory], dict[str, Any]]: adv = parse_cert_tr_email(raw) if not adv: continue - # Sender filter (allowlist) - parser already extracts from raw header via raw["from"] - # Need to re-check using parsed adv's author_login / raw sender from_hdr = adv.raw.get("from") or adv.author_login or "" if allowlist and not is_cert_tr_sender(from_hdr, allowlist): diag["filtered_sender"] += 1 continue - # Also filter by folder? No, folder already limits all_advs.append(adv) diag["parsed"] += 1 - except Exception as e: + except (ValueError, TypeError, AttributeError) as e: logger.debug("parse_cert_tr_email failed: %s", e, exc_info=True) continue diff --git a/src/advisory_rss/cli.py b/src/advisory_rss/cli.py index 89b2de9..29aff49 100644 --- a/src/advisory_rss/cli.py +++ b/src/advisory_rss/cli.py @@ -1,6 +1,5 @@ from __future__ import annotations -# ruff: noqa: BLE001, S110 import asyncio import getpass import logging @@ -44,7 +43,8 @@ def configure_logging( # Prefer explicit args, else settings lf = log_file if log_file is not None else getattr(settings, "log_file", None) fmt = log_format if log_format != "text" else getattr(settings, "log_format", "text") - except Exception: + except (OSError, ValueError, RuntimeError, AttributeError) as e: + logger.debug("Failed to load settings for logging config: %s", e) lf = log_file fmt = log_format _setup_logging(level, log_file=lf, log_format=fmt, force=force, use_stderr=use_stderr) @@ -188,7 +188,7 @@ async def _run() -> int: click.echo(f"Latest GitHub: {advs[0].ghsa_id} - {advs[0].summary[:80]}") except AuthError as e: msg = _redact(str(e)) - logger.error("GitHub sync auth error: %s", msg, exc_info=True) + logger.exception("GitHub sync auth error: %s", msg) click.echo(f"GitHub Auth error (401): {msg}", err=True) cache.mark_error(msg) if source == "github": @@ -209,7 +209,7 @@ async def _run() -> int: combined_diag["errors"].append(msg) except (OSError, ValueError, RuntimeError, httpx.HTTPError) as e: msg = _redact(str(e)) - logger.error("GitHub sync failed: %s", msg, exc_info=True) + logger.exception("GitHub sync failed: %s", msg) click.echo(f"GitHub sync failed: {msg}", err=True) cache.mark_error(msg) if source == "github": @@ -240,7 +240,7 @@ async def _run() -> int: click.echo(f" CERT-TR warn: {_redact(e)}", err=True) except (OSError, ValueError, RuntimeError) as e: msg = _redact(str(e)) - logger.error("CERT-TR sync failed: %s", msg, exc_info=True) + logger.exception("CERT-TR sync failed: %s", msg) click.echo(f"CERT-TR sync failed: {msg}", err=True) cache.mark_error(msg) if source == "cert-tr": @@ -280,8 +280,8 @@ async def _run() -> int: if combined_diag.get("cert_tr"): cache.set_meta("cert_tr_diag", json.dumps(combined_diag["cert_tr"], ensure_ascii=False)) - except Exception: - pass + except (OSError, ValueError, TypeError, RuntimeError) as e: + logger.debug("Failed to persist cert_tr_diag: %s", e) click.echo(f"Sync OK - {count} advisories upserted total ({len(all_advisories)} fetched before dedup)") if combined_diag["errors"]: click.echo(f"Warnings ({len(combined_diag['errors'])}):", err=True) @@ -491,7 +491,7 @@ def serve( use_stderr=False, ) logger.info("daemon child logging reconfigured file=%s", log_file) - except Exception as e: + except (OSError, ValueError, RuntimeError) as e: print(f"daemon logging reconfigure failed: {e}", file=sys.stderr) # Import uvicorn late @@ -620,8 +620,8 @@ def _status() -> None: cert_c = sum(1 for a in all_adv if getattr(a, "source", "") == "cert-tr") click.echo(f" - GitHub: {github_c}") click.echo(f" - CERT-TR: {cert_c}") - except Exception: - pass + except (OSError, ValueError, RuntimeError) as e: + logger.debug("Failed to load per-source breakdown: %s", e) click.echo(f"Last sync: {meta.last_successful_sync or 'never'}") click.echo(f"Next sync: {meta.next_scheduled_sync or 'not scheduled'}") click.echo(f"Rate limited until: {meta.rate_limited_until or 'no'}") @@ -633,11 +633,12 @@ def _status() -> None: click.echo(f"Max items (RSS): {settings.max_items}") click.echo(f"CERT-TR enabled: {settings.enable_cert_tr}") if settings.enable_cert_tr: - accs = settings.get_proton_accounts() + accs = settings.get_cert_tr_accounts() click.echo(f"CERT-TR accounts: {len(accs)}") for a in accs: # Never show password - click.echo(f" - {a['email']} -> folder={a['folder']} host={a['host']}:{a['port']} sec={a['security']}") + prov = a.get("provider", "unknown") + click.echo(f" - [{prov}] {a['email']} -> folder={a['folder']} host={a['host']}:{a['port']} sec={a['security']}") click.echo(f"CERT-TR allowlist: {settings.cert_tr_sender_allowlist}") click.echo(f"CERT-TR max mails: {settings.cert_tr_max_mails} search_days: {settings.cert_tr_search_days or 'all'}") diag_raw = cache.get_meta("cert_tr_diag") diff --git a/src/advisory_rss/config/constants.py b/src/advisory_rss/config/constants.py index 2664cd3..7697144 100644 --- a/src/advisory_rss/config/constants.py +++ b/src/advisory_rss/config/constants.py @@ -30,12 +30,16 @@ GITHUB_API_VERSION = "2022-11-28" # Pinned to 2022-11-28 (stable). 2026-03-10 also valid but not required. -# Cert-TR / Proton +# Cert-TR / Proton + Gmail DEFAULT_CERT_TR_ENABLED = False DEFAULT_PROTON_IMAP_HOST = "127.0.0.1" DEFAULT_PROTON_IMAP_PORT = 1143 DEFAULT_PROTON_IMAP_SECURITY = "STARTTLS" DEFAULT_PROTON_FOLDER = "INBOX" +DEFAULT_GMAIL_IMAP_HOST = "imap.gmail.com" +DEFAULT_GMAIL_IMAP_PORT = 993 +DEFAULT_GMAIL_IMAP_SECURITY = "SSL" +DEFAULT_GMAIL_FOLDER = "INBOX" DEFAULT_CERT_TR_SENDER_ALLOWLIST = "siberguvenlik.gov.tr" # Security diff --git a/src/advisory_rss/config/settings.py b/src/advisory_rss/config/settings.py index 089cdf5..b6a60c7 100644 --- a/src/advisory_rss/config/settings.py +++ b/src/advisory_rss/config/settings.py @@ -15,6 +15,10 @@ DEFAULT_CERT_TR_SENDER_ALLOWLIST, DEFAULT_FILTER_MODE, DEFAULT_GITHUB_API_BASE, + DEFAULT_GMAIL_FOLDER, + DEFAULT_GMAIL_IMAP_HOST, + DEFAULT_GMAIL_IMAP_PORT, + DEFAULT_GMAIL_IMAP_SECURITY, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL, DEFAULT_MAX_ITEMS, @@ -133,6 +137,31 @@ def __init__( proton_imap_folders: str | None = Field( default=None, validation_alias="PROTON_IMAP_FOLDERS" ) + # Gmail (alternative IMAP for CERT-TR, e.g., imap.gmail.com) + gmail_email: str | None = Field(default=None, validation_alias="GMAIL_EMAIL") + gmail_app_password: str | None = Field( + default=None, validation_alias="GMAIL_APP_PASSWORD" + ) + gmail_password: str | None = Field(default=None, validation_alias="GMAIL_PASSWORD") + gmail_imap_host: str = Field( + default=DEFAULT_GMAIL_IMAP_HOST, validation_alias="GMAIL_IMAP_HOST" + ) + gmail_imap_port: int = Field( + default=DEFAULT_GMAIL_IMAP_PORT, validation_alias="GMAIL_IMAP_PORT" + ) + gmail_imap_security: str = Field( + default=DEFAULT_GMAIL_IMAP_SECURITY, validation_alias="GMAIL_IMAP_SECURITY" + ) + gmail_imap_folder: str = Field( + default=DEFAULT_GMAIL_FOLDER, validation_alias="GMAIL_IMAP_FOLDER" + ) + gmail_emails: str | None = Field(default=None, validation_alias="GMAIL_EMAILS") + gmail_app_passwords: str | None = Field( + default=None, validation_alias="GMAIL_APP_PASSWORDS" + ) + gmail_imap_folders: str | None = Field( + default=None, validation_alias="GMAIL_IMAP_FOLDERS" + ) # Optional: per-account JSON-like override not needed; plural fields suffice cert_tr_sender_allowlist: str = Field( default=DEFAULT_CERT_TR_SENDER_ALLOWLIST, @@ -286,12 +315,12 @@ def _validate_proton_folder(cls, v: str) -> str: fv = (v or DEFAULT_PROTON_FOLDER).strip() if not fv: return DEFAULT_PROTON_FOLDER - # Allow IMAP hierarchy chars: letters, digits, /, ., -, _, space + # Allow IMAP hierarchy chars: letters, digits, /, ., -, _, space, brackets for Gmail import re if len(fv) > 100: raise ValueError("PROTON_IMAP_FOLDER too long") - if not re.match(r"^[\w .\-/]+$", fv): + if not re.match(r"^[\w .\-/\[\]]+$", fv): raise ValueError(f"PROTON_IMAP_FOLDER {fv!r} contains invalid chars") return fv @@ -305,13 +334,56 @@ def _validate_proton_folders(cls, v: str | None) -> str | None: raw = str(v).strip() if len(raw) > 500: raise ValueError("PROTON_IMAP_FOLDERS too long") - # split and validate each - parts = [p.strip() for p in re.split(r"[,\s;]+", raw) if p.strip()] + # split only on comma/semicolon to preserve spaces/brackets inside folder names + parts = [p.strip() for p in re.split(r"[;,]+", raw) if p.strip()] + # if no comma/semicolon but spaces, fallback to comma/whitespace split for backward compat + if len(parts) == 1 and "," not in raw and ";" not in raw: + parts = [p.strip() for p in re.split(r"[,\s;]+", raw) if p.strip()] for p in parts: - if len(p) > 100 or not re.match(r"^[\w .\-/]+$", p): + if len(p) > 100 or not re.match(r"^[\w .\-/\[\]]+$", p): raise ValueError(f"PROTON_IMAP_FOLDERS entry {p!r} invalid") return v + @field_validator("gmail_imap_security") + @classmethod + def _validate_gmail_security(cls, v: str) -> str: + lvl = (v or DEFAULT_GMAIL_IMAP_SECURITY).strip().upper() + if lvl not in {"STARTTLS", "SSL", "NONE"}: + raise ValueError("GMAIL_IMAP_SECURITY must be STARTTLS|SSL|NONE") + return lvl + + @field_validator("gmail_imap_folder") + @classmethod + def _validate_gmail_folder(cls, v: str) -> str: + fv = (v or DEFAULT_GMAIL_FOLDER).strip() + if not fv: + return DEFAULT_GMAIL_FOLDER + import re + + if len(fv) > 100: + raise ValueError("GMAIL_IMAP_FOLDER too long") + if not re.match(r"^[\w .\-/\[\]]+$", fv): + raise ValueError(f"GMAIL_IMAP_FOLDER {fv!r} contains invalid chars") + return fv + + @field_validator("gmail_imap_folders") + @classmethod + def _validate_gmail_folders(cls, v: str | None) -> str | None: + if v is None or not str(v).strip(): + return v + import re + + raw = str(v).strip() + if len(raw) > 500: + raise ValueError("GMAIL_IMAP_FOLDERS too long") + parts = [p.strip() for p in re.split(r"[;,]+", raw) if p.strip()] + if len(parts) == 1 and "," not in raw and ";" not in raw: + parts = [p.strip() for p in re.split(r"[,\s;]+", raw) if p.strip()] + for p in parts: + if len(p) > 100 or not re.match(r"^[\w .\-/\[\]]+$", p): + raise ValueError(f"GMAIL_IMAP_FOLDERS entry {p!r} invalid") + return v + @field_validator("cert_tr_max_mails") @classmethod def _validate_cert_tr_max(cls, v: int) -> int: @@ -423,6 +495,95 @@ def get_proton_accounts(self) -> list[dict[str, str]]: ) return out + def get_gmail_accounts(self) -> list[dict[str, str]]: + """Resolve Gmail IMAP accounts for CERT-TR. + + Supports single (GMAIL_EMAIL + GMAIL_APP_PASSWORD) and + multi (GMAIL_EMAILS, GMAIL_APP_PASSWORDS comma-separated, + GMAIL_IMAP_FOLDERS). Host/port/security are shared per Gmail + config (defaults imap.gmail.com:993 SSL). + Each account is {email,password,folder,host,port,security,provider}. + """ + emails: list[str] = [] + passwords: list[str] = [] + folders: list[str] = [] + + # Helper to split passwords preserving internal spaces (comma-only) + def _split_passwords(raw: str | None) -> list[str]: + if not raw or not raw.strip(): + return [] + # split only on comma, keep internal spaces + parts = [p.strip() for p in raw.split(",")] + return [p for p in parts if p] + + if self.gmail_emails and self.gmail_emails.strip(): + emails = self._split_list(self.gmail_emails) + if self.gmail_app_passwords: + passwords = _split_passwords(self.gmail_app_passwords) + elif self.gmail_app_password or self.gmail_password: + # single fallback handled later + pass + if self.gmail_imap_folders: + folders = self._split_list(self.gmail_imap_folders) + elif self.gmail_email and self.gmail_email.strip(): + emails = [self.gmail_email.strip()] + # Prefer app password, fallback to GMAIL_PASSWORD alias + pw = self.gmail_app_password or self.gmail_password + if pw: + passwords = [pw.strip()] + if self.gmail_imap_folders and self.gmail_imap_folders.strip(): + folders = self._split_list(self.gmail_imap_folders) + else: + folders = [self.gmail_imap_folder.strip() or DEFAULT_GMAIL_FOLDER] + else: + return [] + + out: list[dict[str, str]] = [] + default_folder = (self.gmail_imap_folder or DEFAULT_GMAIL_FOLDER).strip() or DEFAULT_GMAIL_FOLDER + if len(passwords) == 1 and len(emails) > 1: + passwords = passwords * len(emails) + for idx, email in enumerate(emails): + if not email or "@" not in email: + continue + pw = "" + if idx < len(passwords): + pw = passwords[idx] + elif passwords: + pw = passwords[0] + else: + # fallback single + single_pw = self.gmail_app_password or self.gmail_password + pw = single_pw or "" + folder = default_folder + if idx < len(folders) and folders[idx]: + folder = folders[idx] + folder = folder.strip() or default_folder + out.append( + { + "email": email.strip(), + "password": pw.strip() if pw else "", + "folder": folder, + "host": (self.gmail_imap_host or DEFAULT_GMAIL_IMAP_HOST).strip(), + "port": str(self.gmail_imap_port), + "security": (self.gmail_imap_security or DEFAULT_GMAIL_IMAP_SECURITY).strip().upper(), + "provider": "gmail", + } + ) + return out + + def get_cert_tr_accounts(self) -> list[dict[str, str]]: + """Unified CERT-TR IMAP accounts: Proton + Gmail. + + Each entry is {email,password,folder,host,port,security,provider}. + Proton entries have provider 'proton', Gmail 'gmail'. + """ + proton = self.get_proton_accounts() + for p in proton: + p.setdefault("provider", "proton") + gmail = self.get_gmail_accounts() + # gmail already has provider + return proton + gmail + def cert_tr_sender_list(self) -> list[str]: if not self.cert_tr_sender_allowlist or not self.cert_tr_sender_allowlist.strip(): return [DEFAULT_CERT_TR_SENDER_ALLOWLIST] diff --git a/src/advisory_rss/logging_config.py b/src/advisory_rss/logging_config.py index bcacf40..bd5b1d2 100644 --- a/src/advisory_rss/logging_config.py +++ b/src/advisory_rss/logging_config.py @@ -18,6 +18,8 @@ _TOKEN_RE = re.compile(TOKEN_REDACT_PATTERN) +logger = logging.getLogger(__name__) + def _redact(s: str) -> str: return _TOKEN_RE.sub("***", s) @@ -142,7 +144,8 @@ def format(self, record: logging.LogRecord) -> str: extras[k] = _redact(v) else: extras[k] = v - except Exception: + except (AttributeError, TypeError, ValueError, RuntimeError) as e: + logger.debug("Redact extra %s failed: %s", k, e) extras[k] = str(v) if extras: payload["extra"] = extras @@ -226,8 +229,8 @@ def setup_logging( root.removeHandler(h) try: h.close() - except Exception: - pass + except (OSError, ValueError, RuntimeError) as e: + logger.debug("Handler close failed: %s", e) # Choose formatter if fmt_str == "json": @@ -341,5 +344,5 @@ def reset_logging_for_tests() -> None: root.removeHandler(h) try: h.close() - except Exception: - pass + except (OSError, ValueError, RuntimeError) as e: + logger.debug("Handler close failed in reset: %s", e) diff --git a/src/advisory_rss/server/app.py b/src/advisory_rss/server/app.py index 202d5d6..d0cc80b 100644 --- a/src/advisory_rss/server/app.py +++ b/src/advisory_rss/server/app.py @@ -1,5 +1,4 @@ # FastAPI app factory -# ruff: noqa: BLE001, S110 from __future__ import annotations @@ -118,7 +117,8 @@ async def health() -> JSONResponse: all_advs = cache.load_all() github_count = sum(1 for a in all_advs if (getattr(a, "source", "github") or "github") == "github") cert_tr_count = sum(1 for a in all_advs if getattr(a, "source", "") == "cert-tr") - except Exception: + except (OSError, ValueError, RuntimeError, AttributeError) as e: + logger.debug("Failed to load per-source counts: %s", e) github_count = None cert_tr_count = None # cert_tr diag @@ -129,8 +129,8 @@ async def health() -> JSONResponse: raw = cache.get_meta("cert_tr_diag") if raw: cert_tr_diag = json.loads(raw) - except Exception: - pass + except (OSError, ValueError, TypeError, RuntimeError) as e: + logger.debug("Failed to load cert_tr_diag: %s", e) return JSONResponse( content={ "status": "ok" if not meta.rate_limited_until else "rate_limited", @@ -138,7 +138,7 @@ async def health() -> JSONResponse: "github_count": github_count, "cert_tr_count": cert_tr_count, "cert_tr_enabled": settings.enable_cert_tr, - "cert_tr_accounts": len(settings.get_proton_accounts()) if settings.enable_cert_tr else 0, + "cert_tr_accounts": len(settings.get_cert_tr_accounts()) if settings.enable_cert_tr else 0, "cert_tr_diag": cert_tr_diag, "last_successful_sync": meta.last_successful_sync, "next_scheduled_sync": meta.next_scheduled_sync, @@ -308,8 +308,8 @@ async def refresh(request: Request) -> JSONResponse: if diag_comb.get("cert_tr"): cache.set_meta("cert_tr_diag", json.dumps(diag_comb["cert_tr"], ensure_ascii=False)) - except Exception: - pass + except (OSError, ValueError, TypeError, RuntimeError) as e: + logger.debug("Failed to persist cert_tr_diag: %s", e) logger.info("POST /refresh ok: count=%d", count, extra={"diag": diag_comb}) return JSONResponse( content={ @@ -333,7 +333,6 @@ async def unhandled(request: Request, exc: Exception): "Unhandled error on %s: %s", request.url.path, _redact(str(exc)), - exc_info=True, ) return JSONResponse( status_code=500, @@ -366,9 +365,9 @@ async def background_refresh_loop(settings: Settings, cache: CacheStore) -> None if not token and not settings.enable_cert_tr: logger.info("Background refresh skipped - no token and CERT-TR disabled") continue - # Also skip if CERT-TR enabled but no Proton accounts and no token => nothing to do - if not token and settings.enable_cert_tr and not settings.get_proton_accounts(): - logger.info("Background refresh skipped - CERT-TR enabled but no Proton accounts and no GitHub token") + # Also skip if CERT-TR enabled but no IMAP accounts and no token => nothing to do + if not token and settings.enable_cert_tr and not settings.get_cert_tr_accounts(): + logger.info("Background refresh skipped - CERT-TR enabled but no IMAP accounts and no GitHub token") continue try: all_advs = [] @@ -423,13 +422,14 @@ def _fetch_ct(): if cert_diag is not None: cache.set_meta("cert_tr_diag", json.dumps(cert_diag, ensure_ascii=False)) - except Exception: - pass + except (OSError, ValueError, TypeError, RuntimeError) as e: + logger.debug("Failed to persist cert_diag in background: %s", e) gh_count = 0 if isinstance(github_diag, dict): try: gh_count = int(github_diag.get("filtered_count") or 0) - except Exception: + except (ValueError, TypeError) as e: + logger.debug("Failed to parse gh_count: %s", e) gh_count = 0 ct_count = int(cert_diag.get("parsed") or 0) if isinstance(cert_diag, dict) else 0 logger.info( diff --git a/tests/unit/test_cert_tr_imap.py b/tests/unit/test_cert_tr_imap.py index adc6781..fea713c 100644 --- a/tests/unit/test_cert_tr_imap.py +++ b/tests/unit/test_cert_tr_imap.py @@ -1,3 +1,4 @@ +import imaplib from unittest.mock import MagicMock, patch from advisory_rss.cert_tr.imap import fetch_raw_emails @@ -54,7 +55,49 @@ def test_fetch_multi_account_distinct_folders(): m1 = _mock_imap([SAMPLE_RAW]) m2 = _mock_imap([SAMPLE_RAW]) mock_connect.side_effect = [m1, m2] - r1 = fetch_raw_emails(accs[0], ["siberguvenlik.gov.tr"], max_mails=1) - r2 = fetch_raw_emails(accs[1], ["siberguvenlik.gov.tr"], max_mails=1) + assert fetch_raw_emails(accs[0], ["siberguvenlik.gov.tr"], max_mails=1) is not None + assert fetch_raw_emails(accs[1], ["siberguvenlik.gov.tr"], max_mails=1) is not None assert m1.select.call_args[0][0] == "INBOX" assert m2.select.call_args[0][0] == "INBOX.CERT-TR" + + +def test_gmail_host_allowed_and_proton_loopback(): + from advisory_rss.cert_tr.imap import ALLOWED_HOSTS, connect_imap + + assert "imap.gmail.com" in ALLOWED_HOSTS + assert "127.0.0.1" in ALLOWED_HOSTS + + # Gmail host should be allowed (mock network to avoid real connection) + with patch("advisory_rss.cert_tr.imap.imaplib.IMAP4_SSL") as mock_ssl: + mock_inst = MagicMock() + mock_inst.login.side_effect = imaplib.IMAP4.error("mock auth fail") + mock_ssl.return_value = mock_inst + try: + connect_imap( + { + "host": "imap.gmail.com", + "port": "993", + "security": "SSL", + "email": "test@gmail.com", + "password": "bad", + } + ) + except imaplib.IMAP4.error: + pass # expected login failure, host was allowed + except ValueError as ve: + raise AssertionError(f"imap.gmail.com should be allowed, got {ve}") from ve + + # Evil host should be rejected before network + try: + connect_imap( + { + "host": "evil.com", + "port": "993", + "security": "SSL", + "email": "test@gmail.com", + "password": "bad", + } + ) + raise AssertionError("evil host should be rejected") + except ValueError as ve: + assert "IMAP host" in str(ve) diff --git a/tests/unit/test_cert_tr_settings.py b/tests/unit/test_cert_tr_settings.py index 645c8fe..6a397ca 100644 --- a/tests/unit/test_cert_tr_settings.py +++ b/tests/unit/test_cert_tr_settings.py @@ -50,3 +50,57 @@ def test_sender_allowlist(): s = Settings(_env_file="/dev/null", CERT_TR_SENDER_ALLOWLIST="siberguvenlik.gov.tr, usom.gov.tr", _env_file_encoding="utf-8") assert "siberguvenlik.gov.tr" in s.cert_tr_sender_list() assert "usom.gov.tr" in s.cert_tr_sender_list() + + +def test_gmail_single(): + s = Settings( + _env_file="/dev/null", + ENABLE_CERT_TR="true", + GMAIL_EMAIL="a@gmail.com", + GMAIL_APP_PASSWORD="abcd efgh ijkl mnop", + GMAIL_IMAP_FOLDER="INBOX", + _env_file_encoding="utf-8", + ) + accs = s.get_gmail_accounts() + assert len(accs) == 1 + assert accs[0]["email"] == "a@gmail.com" + assert accs[0]["password"] == "abcd efgh ijkl mnop" + assert accs[0]["host"] == "imap.gmail.com" + assert accs[0]["security"] == "SSL" + + +def test_gmail_multi_with_spaces(): + s = Settings( + _env_file="/dev/null", + ENABLE_CERT_TR="true", + GMAIL_EMAILS="a@gmail.com,b@gmail.com", + GMAIL_APP_PASSWORDS="pass1,pass2 with spaces", + GMAIL_IMAP_FOLDERS="INBOX,INBOX.CERT", + _env_file_encoding="utf-8", + ) + accs = s.get_gmail_accounts() + assert len(accs) == 2 + assert accs[0]["password"] == "pass1" + assert accs[1]["password"] == "pass2 with spaces" + assert accs[1]["folder"] == "INBOX.CERT" + + +def test_mixed_proton_gmail_unified(): + s = Settings( + _env_file="/dev/null", + ENABLE_CERT_TR="true", + PROTON_BRIDGE_EMAIL="p@proton.me", + PROTON_BRIDGE_PASSWORD="pp", + GMAIL_EMAIL="g@gmail.com", + GMAIL_APP_PASSWORD="gp", + _env_file_encoding="utf-8", + ) + accs = s.get_cert_tr_accounts() + assert len(accs) == 2 + providers = {a["provider"] for a in accs} + assert providers == {"proton", "gmail"} + # proton should be loopback, gmail imap.gmail.com + proton = next(a for a in accs if a["provider"] == "proton") + gmail = next(a for a in accs if a["provider"] == "gmail") + assert proton["host"] == "127.0.0.1" + assert gmail["host"] == "imap.gmail.com" From 914610a84442f9684d47b672f81360ce8401bd77 Mon Sep 17 00:00:00 2001 From: Lunixizm0 <216941080+Lunixizm0@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:48:26 +0300 Subject: [PATCH 3/4] feat: Implement Gmail OAuth support for IMAP authentication --- pyproject.toml | 3 +- src/advisory_rss/auth/gmail.py | 248 +++++++++++++++ src/advisory_rss/cert_tr/imap.py | 133 +++++++- src/advisory_rss/cli.py | 451 ++++++++++++++++++++++++--- src/advisory_rss/config/constants.py | 6 + src/advisory_rss/config/settings.py | 235 ++++++++++++-- 6 files changed, 991 insertions(+), 85 deletions(-) create mode 100644 src/advisory_rss/auth/gmail.py diff --git a/pyproject.toml b/pyproject.toml index 1a27fe6..18b3049 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,8 @@ dependencies = [ "pydantic-settings>=2.4", "python-dotenv>=1.0", "click>=8.0", - + "google-auth>=2.30", + "google-auth-oauthlib>=1.1", ] [project.optional-dependencies] diff --git a/src/advisory_rss/auth/gmail.py b/src/advisory_rss/auth/gmail.py new file mode 100644 index 0000000..8009b5c --- /dev/null +++ b/src/advisory_rss/auth/gmail.py @@ -0,0 +1,248 @@ +"""Gmail OAuth helpers for CERT-TR IMAP (XOAUTH2).""" + +from __future__ import annotations + +import base64 +import json +import logging +from pathlib import Path +from typing import Any + +import httpx + +from advisory_rss.config.constants import ( + DEFAULT_GMAIL_OAUTH_CACHE, + GMAIL_OAUTH_SCOPES, + GMAIL_OAUTH_TOKEN_URI, +) + +logger = logging.getLogger(__name__) + + +def _load_oauth_file(path: str | Path) -> dict[str, Any]: + p = Path(path) + if not p.exists(): + return {} + try: + data = json.loads(p.read_text(encoding="utf-8")) + return data if isinstance(data, dict) else {} + except (OSError, ValueError, json.JSONDecodeError) as e: + logger.debug("Failed to load Gmail OAuth file %s: %s", path, e) + return {} + + +def _save_oauth_file(path: str | Path, data: dict[str, Any]) -> None: + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + # Restrict permissions (600) for token file + try: + p.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + try: + p.chmod(0o600) + except OSError as e: + logger.debug("chmod failed for %s: %s", p, e) + except OSError as e: + logger.warning("Failed to save Gmail OAuth file %s: %s", p, e) + raise + + +def save_gmail_oauth_token( + token_file: str | Path, + email: str, + refresh_token: str, + client_id: str, + client_secret: str, + access_token: str | None = None, + expiry: str | None = None, +) -> None: + data = _load_oauth_file(token_file) + # Preserve top-level client_id/secret if not per-account + if client_id: + data["client_id"] = client_id + if client_secret: + data["client_secret"] = client_secret + if "accounts" not in data or not isinstance(data["accounts"], dict): + data["accounts"] = {} + acct = data["accounts"].get(email, {}) + if not isinstance(acct, dict): + acct = {} + acct["refresh_token"] = refresh_token + if access_token: + acct["access_token"] = access_token + if expiry: + acct["expiry"] = expiry + data["accounts"][email] = acct + _save_oauth_file(token_file, data) + logger.info("Saved Gmail OAuth token for %s to %s", email, token_file) + + +def get_refresh_token_for_email( + email: str, + settings: Any, +) -> tuple[str | None, str | None, str | None]: + """Return (refresh_token, client_id, client_secret) for email. + + Checks env vars first (GMAIL_OAUTH_REFRESH_TOKEN etc), then file. + For multi-account, checks GMAIL_OAUTH_REFRESH_TOKENS aligned with GMAIL_EMAILS. + """ + # Check env single/multi + # settings may have gmail_oauth_refresh_token(s) and client_id/secret + try: + # Use get_gmail_accounts logic? But we can directly check settings + # Check file first for per-account + file_data = _load_oauth_file(getattr(settings, "gmail_oauth_token_file", DEFAULT_GMAIL_OAUTH_CACHE)) + # Try file per-account + accounts = file_data.get("accounts") if isinstance(file_data, dict) else None + if isinstance(accounts, dict) and email in accounts: + info = accounts[email] + if isinstance(info, dict) and info.get("refresh_token"): + cid = file_data.get("client_id") or getattr(settings, "gmail_oauth_client_id", None) + csec = file_data.get("client_secret") or getattr(settings, "gmail_oauth_client_secret", None) + return str(info["refresh_token"]), str(cid or "") if cid else None, str(csec or "") if csec else None + if isinstance(info, str): + cid = file_data.get("client_id") or getattr(settings, "gmail_oauth_client_id", None) + csec = file_data.get("client_secret") or getattr(settings, "gmail_oauth_client_secret", None) + return info, str(cid or "") if cid else None, str(csec or "") if csec else None + # Check flat file + if isinstance(file_data, dict) and email in file_data and isinstance(file_data[email], str): + cid = file_data.get("client_id") or getattr(settings, "gmail_oauth_client_id", None) + csec = file_data.get("client_secret") or getattr(settings, "gmail_oauth_client_secret", None) + return str(file_data[email]), str(cid or "") if cid else None, str(csec or "") if csec else None + + # Check env multi + gmail_emails = getattr(settings, "gmail_emails", None) + gmail_refresh_tokens = getattr(settings, "gmail_oauth_refresh_tokens", None) + if gmail_emails and gmail_refresh_tokens: + # _split_list is private, but we can emulate + import re + + emails = [e.strip() for e in re.split(r"[,\s;]+", gmail_emails) if e.strip()] + # For refresh tokens, split on comma only (preserve spaces? refresh tokens have no spaces) + rts = [r.strip() for r in gmail_refresh_tokens.split(",") if r.strip()] + try: + idx = [e.lower() for e in emails].index(email.lower()) + if idx < len(rts): + cid = getattr(settings, "gmail_oauth_client_id", None) + csec = getattr(settings, "gmail_oauth_client_secret", None) + return rts[idx], str(cid or "") if cid else None, str(csec or "") if csec else None + except ValueError: + pass + + # Check env single + single_rt = getattr(settings, "gmail_oauth_refresh_token", None) + if single_rt and single_rt.strip(): + # If single email matches or only one email configured + # If settings has single gmail_email matching, use it; else if only one refresh token and email matches gmail_email + gmail_email = getattr(settings, "gmail_email", None) + if gmail_email and gmail_email.strip().lower() == email.lower(): + cid = getattr(settings, "gmail_oauth_client_id", None) + csec = getattr(settings, "gmail_oauth_client_secret", None) + return single_rt.strip(), str(cid or "") if cid else None, str(csec or "") if csec else None + # If no specific email, but we have refresh token and email is the one being queried, return it + # This handles case where get_gmail_accounts already built with refresh token + cid = getattr(settings, "gmail_oauth_client_id", None) + csec = getattr(settings, "gmail_oauth_client_secret", None) + # Only return if email is in gmail accounts list or if no list + # For safety, return if we have a refresh token and no other mapping + if not gmail_emails: + return single_rt.strip(), str(cid or "") if cid else None, str(csec or "") if csec else None + except (OSError, ValueError, AttributeError) as e: + logger.debug("get_refresh_token failed for %s: %s", email, e) + return None, None, None + + +def fetch_access_token( + refresh_token: str, + client_id: str, + client_secret: str, + token_uri: str = GMAIL_OAUTH_TOKEN_URI, +) -> tuple[str, int | None]: + """Exchange refresh token for access token via Google OAuth2. + + Returns (access_token, expires_in_seconds). + Raises on failure. + """ + if not refresh_token or not client_id or not client_secret: + raise ValueError("Missing OAuth credentials (refresh_token/client_id/secret)") + + data = { + "client_id": client_id, + "client_secret": client_secret, + "refresh_token": refresh_token, + "grant_type": "refresh_token", + } + try: + resp = httpx.post(token_uri, data=data, timeout=15.0) + resp.raise_for_status() + j = resp.json() + access_token = j.get("access_token") + expires_in = j.get("expires_in") + if not access_token: + raise ValueError(f"No access_token in response: {j}") + return str(access_token), int(expires_in) if expires_in else None + except httpx.HTTPError as e: + logger.warning("Failed to fetch Gmail access token: %s", e) + raise + except (ValueError, TypeError) as e: + logger.warning("Invalid token response: %s", e) + raise + + +def build_xoauth2_string(email: str, access_token: str) -> str: + """Build base64-encoded XOAUTH2 string for IMAP AUTHENTICATE.""" + # Format: user=\x01auth=Bearer \x01\x01 + raw = f"user={email}\x01auth=Bearer {access_token}\x01\x01" + return base64.b64encode(raw.encode("utf-8")).decode("utf-8") + + +def run_gmail_oauth_flow( + client_id: str, + client_secret: str, + email: str | None = None, +) -> tuple[str, str, str | None]: + """Run InstalledAppFlow to obtain refresh token. + + Returns (refresh_token, access_token, expiry). + Requires google-auth-oauthlib to be installed. + Opens browser for user consent. + """ + try: + from google_auth_oauthlib.flow import InstalledAppFlow + except ImportError as e: + raise ImportError( + "google-auth-oauthlib is required for Gmail OAuth. Install with: pip install google-auth google-auth-oauthlib" + ) from e + + # Use the same scopes as constants + client_config = { + "installed": { + "client_id": client_id, + "client_secret": client_secret, + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": GMAIL_OAUTH_TOKEN_URI, + "redirect_uris": ["http://localhost", "urn:ietf:wg:oauth:2.0:oob"], + } + } + + flow = InstalledAppFlow.from_client_config(client_config, scopes=GMAIL_OAUTH_SCOPES) + # Optional: set login hint if email provided + # The flow will open browser; we use local server + # Use port 0 to get random available port, or 8080 + # prompt=consent to ensure refresh_token is returned + creds = flow.run_local_server( + port=0, + prompt="consent", + access_type="offline", + include_granted_scopes="true", + ) + refresh_token = getattr(creds, "refresh_token", None) + access_token = getattr(creds, "token", None) + expiry = getattr(creds, "expiry", None) + expiry_str = expiry.isoformat() if expiry else None + if not refresh_token: + # Sometimes refresh_token is only returned on first consent; if not, we try to use existing + # But we should warn + logger.warning("OAuth flow did not return a refresh_token (maybe already consented). Try revoking access and retrying.") + # Fallback: try to use current refresh token if available? For now, raise + raise ValueError("No refresh_token returned from OAuth flow. Ensure you revoke previous consent in Google Account and retry with prompt=consent.") + return str(refresh_token), str(access_token or ""), expiry_str diff --git a/src/advisory_rss/cert_tr/imap.py b/src/advisory_rss/cert_tr/imap.py index 12a0578..6c6e628 100644 --- a/src/advisory_rss/cert_tr/imap.py +++ b/src/advisory_rss/cert_tr/imap.py @@ -8,6 +8,8 @@ import time from datetime import UTC, datetime, timedelta +import httpx + logger = logging.getLogger(__name__) # Allowed IMAP hosts: Proton Bridge loopback + Gmail @@ -42,6 +44,10 @@ def connect_imap(account: dict[str, str]) -> imaplib.IMAP4: security = (account.get("security") or "STARTTLS").upper() email = account.get("email", "") password = account.get("password", "") + auth_method = account.get("auth_method", "password") + oauth_refresh_token = account.get("oauth_refresh_token", "") + oauth_client_id = account.get("oauth_client_id", "") + oauth_client_secret = account.get("oauth_client_secret", "") if host not in ALLOWED_HOSTS: raise ValueError(f"IMAP host must be one of {sorted(ALLOWED_HOSTS)}, got {host!r}") @@ -58,15 +64,120 @@ def connect_imap(account: dict[str, str]) -> imaplib.IMAP4: else: # NONE mail = imaplib.IMAP4(host, port) - try: - mail.login(email, password) - except imaplib.IMAP4.error as e: - logger.error("IMAP login failed for %s: %s", email, e) - raise + # Gmail OAuth (XOAUTH2) if configured + if auth_method == "oauth" and oauth_refresh_token: + # Need client_id/secret + if not oauth_client_id or not oauth_client_secret: + raise ValueError(f"Gmail OAuth for {email} missing client_id/secret") + try: + from advisory_rss.auth.gmail import build_xoauth2_string, fetch_access_token + + access_token, _ = fetch_access_token( + oauth_refresh_token, oauth_client_id, oauth_client_secret + ) + xoauth2 = build_xoauth2_string(email, access_token) + + # imaplib authenticate with XOAUTH2 + def _auth_cb(_: bytes) -> str: + return xoauth2 + + mail.authenticate("XOAUTH2", _auth_cb) # type: ignore[arg-type] + except (OSError, ValueError, RuntimeError, imaplib.IMAP4.error, httpx.HTTPError, ImportError) as e: + logger.error("IMAP XOAUTH2 failed for %s: %s", email, e) + raise + else: + # Regular password login (Proton Bridge or Gmail App Password) + try: + mail.login(email, password) + except imaplib.IMAP4.error as e: + logger.error("IMAP login failed for %s: %s", email, e) + raise logger.debug("IMAP connected %s folder will be selected later", email) return mail +def _encode_modified_utf7(s: str) -> str: + # IMAP modified UTF-7 (RFC 3501): encode non-ASCII runs as &- where base64 uses , for / + import base64 + + res: list[str] = [] + in_non_ascii = False + buf: list[str] = [] + + def _b64(buf_chars: list[str]) -> str: + b = "".join(buf_chars).encode("utf-16be") + b64 = base64.b64encode(b).decode("ascii") + b64 = b64.replace("/", ",").rstrip("=") + return "&" + b64 + "-" + + for ch in s: + o = ord(ch) + if 0x20 <= o <= 0x7E and ch != "&": + if in_non_ascii: + res.append(_b64(buf)) + buf = [] + in_non_ascii = False + res.append(ch) + elif ch == "&": + if in_non_ascii: + res.append(_b64(buf)) + buf = [] + in_non_ascii = False + res.append("&-") + else: + if not in_non_ascii: + in_non_ascii = True + buf.append(ch) + if in_non_ascii: + res.append(_b64(buf)) + return "".join(res) + + +def _select_folder_imap(mail: imaplib.IMAP4, folder: str, readonly: bool = True) -> tuple[str, list[bytes]]: + # Try UTF-8 first (if server supports UTF8=ACCEPT), then modified UTF-7 + # Gmail supports both, but Proton Bridge is ASCII-only, so INBOX will succeed directly. + # For non-ASCII like "CVE Yazışmaları", try UTF-8, then fallback to modified UTF-7. + # Try with mail._encoding = 'utf-8' + orig_enc = getattr(mail, "_encoding", "ascii") + # First try: enable UTF8 if possible + try: + # Some servers require ENABLE UTF8=ACCEPT before UTF-8 mailbox names + # Ignore failure + try: + mail.enable("UTF8=ACCEPT") + except (imaplib.IMAP4.error, OSError, ValueError): + pass + except (AttributeError, OSError): + pass + # Try UTF-8 + try: + mail._encoding = "utf-8" # type: ignore[attr-defined] + return mail.select(folder, readonly=readonly) # type: ignore[arg-type] + except UnicodeEncodeError: + pass + except imaplib.IMAP4.error: + # Pass to fallback + pass + # Fallback: modified UTF-7 (ASCII) + try: + encoded = _encode_modified_utf7(folder) + mail._encoding = "ascii" # type: ignore[attr-defined] + return mail.select(encoded, readonly=readonly) # type: ignore[arg-type] + except (UnicodeEncodeError, imaplib.IMAP4.error): + # Restore original encoding and re-raise + try: + mail._encoding = orig_enc # type: ignore[attr-defined] + except (AttributeError, OSError): + pass + raise + finally: + # Restore encoding to utf-8 for subsequent commands (search/fetch use ascii criteria) + try: + mail._encoding = "utf-8" # type: ignore[attr-defined] + except (AttributeError, OSError): + pass + + def _build_search_criteria(allowlist: list[str], search_days: int | None) -> list[str]: parts: list[str] = [] if allowlist: @@ -101,19 +212,19 @@ def fetch_raw_emails( typ: str = "" data: list[bytes] = [] try: - typ, data = mail.select(folder, readonly=True) # type: ignore[assignment] - except imaplib.IMAP4.error as e: + typ, data = _select_folder_imap(mail, folder, readonly=True) + except (imaplib.IMAP4.error, UnicodeEncodeError, ValueError) as e: logger.warning("IMAP select folder %r failed for %s: %s", folder, email, e) if folder.upper() != "INBOX": try: - typ2, _data2 = mail.select("INBOX", readonly=True) # type: ignore[assignment] + typ2, _data2 = _select_folder_imap(mail, "INBOX", readonly=True) if typ2 == "OK": logger.info("Fallback to INBOX for %s", email) folder = "INBOX" typ, data = typ2, _data2 # type: ignore[assignment] else: return [] - except (OSError, imaplib.IMAP4.error) as fallback_e: + except (OSError, imaplib.IMAP4.error, UnicodeEncodeError, ValueError) as fallback_e: logger.debug("Fallback select failed for %s: %s", email, fallback_e) return [] else: @@ -158,8 +269,8 @@ def fetch_raw_emails( time.sleep(0.3) try: mail = connect_imap(account) - mail.select(folder, readonly=True) - except (OSError, imaplib.IMAP4.error, ssl.SSLError, ValueError) as e: + _select_folder_imap(mail, folder, readonly=True) + except (OSError, imaplib.IMAP4.error, ssl.SSLError, ValueError, UnicodeEncodeError) as e: logger.warning("IMAP reconnect failed %s batch %d: %s", email, i, e) break diff --git a/src/advisory_rss/cli.py b/src/advisory_rss/cli.py index 29aff49..eb71b3d 100644 --- a/src/advisory_rss/cli.py +++ b/src/advisory_rss/cli.py @@ -61,59 +61,420 @@ def auth() -> None: pass +def _upsert_env_file(key: str, value: str, env_path: str = ".env") -> None: + """Create or update .env file with key=value (preserves other lines).""" + import re as _re2 + + p = Path(env_path) + lines: list[str] = [] + if p.exists(): + try: + lines = p.read_text(encoding="utf-8").splitlines() + except OSError as e: + logger.debug("Failed to read %s: %s", env_path, e) + lines = [] + # Find existing key (case-insensitive for env var name, but keep case) + found = False + new_lines: list[str] = [] + pattern = _re2.compile(rf"^\s*{_re2.escape(key)}\s*=", _re.IGNORECASE) + for line in lines: + if pattern.match(line): + if not found: + new_lines.append(f"{key}={value}") + found = True + # skip duplicate + else: + new_lines.append(line) + if not found: + new_lines.append(f"{key}={value}") + try: + p.write_text("\n".join(new_lines) + "\n", encoding="utf-8") + logger.info("Updated %s with %s", env_path, key) + except OSError as e: + logger.warning("Failed to write %s: %s", env_path, e) + click.echo(f"Warning: could not write {env_path}: {e}", err=True) + + +def _prompt_provider(default: str | None = None) -> str: + choice = click.prompt( + "Hangi kaynak için giriş yapmak istiyorsun?", + type=click.Choice(["github", "proton", "gmail"], case_sensitive=False), + default=default or "github", + show_choices=True, + ) + return str(choice).lower() + + +def _prompt_gmail_method(default: str | None = None) -> str: + choice = click.prompt( + "Gmail için hangi yöntem? [oauth/app-password]", + type=click.Choice(["oauth", "app-password", "app"], case_sensitive=False), + default=default or "oauth", + show_choices=True, + ) + c = str(choice).lower() + return "oauth" if c == "oauth" else "app-password" + + @auth.command("login") -def auth_login() -> None: +@click.option( + "--provider", + type=click.Choice(["github", "proton", "gmail"], case_sensitive=False), + default=None, + help="Hangi kaynak için login (github/proton/gmail). Belirtilmezse interaktif sorulur.", +) +@click.option( + "--method", + type=click.Choice(["oauth", "app-password"], case_sensitive=False), + default=None, + help="Gmail için yöntem (oauth veya app-password). Sadece gmail için.", +) +@click.option("--email", default=None, help="E-posta adresi (proton/gmail için)") +def auth_login(provider: str | None = None, method: str | None = None, email: str | None = None) -> None: settings = get_settings() _setup_logging(settings.log_level, log_file=settings.log_file, log_format=settings.log_format) - logger.info("auth login started") - click.echo("GitHub PAT: fine-grained 'Repository security advisories: Read' is recommended.") - click.echo("Token must be set in .env as GITHUB_TOKEN - credentials file is no longer used.") - existing_env = settings.token - if existing_env: - click.echo(f"Note: GITHUB_TOKEN already set via env ({token_preview(existing_env)}).") - click.echo("If you want to rotate, edit .env directly: GITHUB_TOKEN=...") + logger.info("auth login started provider=%s method=%s email=%s", provider, method, email) - try: - token = getpass.getpass("Paste GitHub PAT (input hidden): ").strip() - except (EOFError, KeyboardInterrupt): - click.echo("\nCancelled.", err=True) - sys.exit(1) - if not token: - click.echo("No token entered - abort.", err=True) - sys.exit(1) + # Determine provider interactively if not provided + if not provider: + provider = _prompt_provider() + provider = provider.lower() - async def _validate() -> dict: - s = get_settings() - cache = CacheStore(s.resolved_cache_path) - client = GitHubClient(s, token, cache=cache) + if provider == "github": + click.echo("GitHub PAT: fine-grained 'Repository security advisories: Read' is recommended.") + click.echo("Token must be set in .env as GITHUB_TOKEN - credentials file is no longer used.") + existing_env = settings.token + if existing_env: + click.echo(f"Note: GITHUB_TOKEN already set via env ({token_preview(existing_env)}).") + click.echo("If you want to rotate, edit .env directly: GITHUB_TOKEN=...") try: - user = await client.get_user() - return user - finally: - await client.close() + token = getpass.getpass("Paste GitHub PAT (input hidden): ").strip() + except (EOFError, KeyboardInterrupt): + click.echo("\nCancelled.", err=True) + sys.exit(1) + if not token: + click.echo("No token entered - abort.", err=True) + sys.exit(1) - try: - user = asyncio.run(_validate()) - login = user.get("login", "unknown") - click.echo(f"+ Validated as @{login}") - except AuthError as e: - click.echo(f"- Authentication failed: {_redact(str(e))}", err=True) - click.echo( - "Check token value, expiry, and scopes (repository_advisories:read).", - err=True, - ) - sys.exit(1) - except (OSError, ValueError, RuntimeError, httpx.HTTPError) as e: - click.echo(f"- Validation error: {_redact(str(e))}", err=True) - sys.exit(1) + async def _validate() -> dict: + s = get_settings() + cache = CacheStore(s.resolved_cache_path) + client = GitHubClient(s, token, cache=cache) + try: + user = await client.get_user() + return user + finally: + await client.close() + + try: + user = asyncio.run(_validate()) + login = user.get("login", "unknown") + click.echo(f"+ Validated as @{login}") + except AuthError as e: + click.echo(f"- Authentication failed: {_redact(str(e))}", err=True) + click.echo( + "Check token value, expiry, and scopes (repository_advisories:read).", + err=True, + ) + sys.exit(1) + except (OSError, ValueError, RuntimeError, httpx.HTTPError) as e: + click.echo(f"- Validation error: {_redact(str(e))}", err=True) + sys.exit(1) + + click.echo("") + click.echo("Token is VALID but NOT saved to disk by this command.") + if click.confirm("Token .env dosyasına yazılsın mı?", default=False): + _upsert_env_file("GITHUB_TOKEN", token) + click.echo("Wrote GITHUB_TOKEN to .env") + else: + click.echo("Set it in .env:") + click.echo(" echo 'GITHUB_TOKEN=YOUR_TOKEN_HERE' >> .env") + click.echo("Then: app sync && app serve") + click.echo("Never commit .env (gitignored).") + return + + if provider == "proton": + # Proton Bridge: email + bridge password + if not email: + email = click.prompt("Proton e-posta adresi", type=str, default=settings.proton_bridge_email or "") + email = email.strip() + if not email or "@" not in email: + click.echo("Geçersiz e-posta", err=True) + sys.exit(1) + # Ask for bridge password (hidden) + try: + bridge_pw = getpass.getpass(f"Proton Bridge şifresi ({email}) (Bridge > Mailbox details, NOT account password): ").strip() + except (EOFError, KeyboardInterrupt): + click.echo("\nCancelled.", err=True) + sys.exit(1) + if not bridge_pw: + click.echo("Şifre girilmedi - abort.", err=True) + sys.exit(1) + # Validate via IMAP + from advisory_rss.cert_tr.imap import connect_imap + + acc = { + "email": email, + "password": bridge_pw, + "folder": settings.proton_imap_folder or "INBOX", + "host": settings.proton_bridge_host or "127.0.0.1", + "port": str(settings.proton_imap_port), + "security": settings.proton_imap_security or "STARTTLS", + } + try: + mail = connect_imap(acc) + try: + mail.select(acc["folder"], readonly=True) + # Try to list folders to verify + mail.list() + finally: + try: + mail.logout() + except (OSError, ValueError, RuntimeError) as e: + logger.debug("Proton logout failed: %s", e) + click.echo(f"+ Proton IMAP doğrulandı: {email} -> {acc['folder']} @ {acc['host']}:{acc['port']}") + except (OSError, ValueError, RuntimeError) as e: + click.echo(f"- Proton IMAP hatası: {_redact(str(e))}", err=True) + click.echo("Bridge çalışıyor mu? Host/port doğru mu? Şifre Bridge şifresi mi?", err=True) + sys.exit(1) + + # Save to .env - handle multi + # If PROTON_BRIDGE_EMAILS already has multiple, append + existing_emails = settings.proton_bridge_emails + if existing_emails and email not in [e.strip() for e in existing_emails.split(",")]: + # Append to multi + new_emails = existing_emails.strip() + f",{email}" + _upsert_env_file("PROTON_BRIDGE_EMAILS", new_emails) + # Passwords + existing_pw = settings.proton_bridge_passwords or "" + if existing_pw.strip(): + new_pw = existing_pw.strip() + f",{bridge_pw}" + else: + # If single was set, migrate to multi + single_pw = settings.proton_bridge_password or "" + if single_pw and single_pw.strip() != bridge_pw: + new_pw = f"{single_pw.strip()},{bridge_pw}" + else: + new_pw = bridge_pw + _upsert_env_file("PROTON_BRIDGE_PASSWORDS", new_pw) + click.echo(f"Wrote PROTON_BRIDGE_EMAILS and PROTON_BRIDGE_PASSWORDS to .env (appended {email})") + else: + _upsert_env_file("PROTON_BRIDGE_EMAIL", email) + _upsert_env_file("PROTON_BRIDGE_PASSWORD", bridge_pw) + click.echo("Wrote PROTON_BRIDGE_EMAIL and PROTON_BRIDGE_PASSWORD to .env") + click.echo("ENABLE_CERT_TR=true olduğundan emin olun, sonra: app sync --source cert-tr") + return + + if provider == "gmail": + # Gmail: ask oauth vs app-password + gmail_method = method + if not gmail_method: + gmail_method = _prompt_gmail_method() + gmail_method = gmail_method.lower() + if gmail_method not in ("oauth", "app-password", "app"): + gmail_method = "oauth" + if gmail_method == "app": + gmail_method = "app-password" + + if not email: + default_email = settings.gmail_email or (settings.gmail_emails.split(",")[0].strip() if settings.gmail_emails else "") + email = click.prompt("Gmail e-posta adresi", type=str, default=default_email) + email = email.strip() + if not email or "@" not in email: + click.echo("Geçersiz e-posta", err=True) + sys.exit(1) + + if gmail_method == "app-password": + try: + app_pw = getpass.getpass(f"Gmail App Password ({email}) (16 haneli, Google Hesabı > Güvenlik > 2FA > Uygulama şifreleri): ").strip() + except (EOFError, KeyboardInterrupt): + click.echo("\nCancelled.", err=True) + sys.exit(1) + if not app_pw: + click.echo("App Password girilmedi", err=True) + sys.exit(1) + # Validate via IMAP (Gmail SSL) + from advisory_rss.cert_tr.imap import connect_imap + + acc = { + "email": email, + "password": app_pw, + "folder": settings.gmail_imap_folder or "INBOX", + "host": settings.gmail_imap_host or "imap.gmail.com", + "port": str(settings.gmail_imap_port), + "security": settings.gmail_imap_security or "SSL", + } + try: + mail = connect_imap(acc) + try: + mail.select(acc["folder"], readonly=True) + finally: + try: + mail.logout() + except (OSError, ValueError, RuntimeError) as e: + logger.debug("Gmail logout failed: %s", e) + click.echo(f"+ Gmail IMAP (App Password) doğrulandı: {email}") + except (OSError, ValueError, RuntimeError) as e: + click.echo(f"- Gmail hatası: {_redact(str(e))}", err=True) + click.echo("App Password doğru mu? 2FA açık mı? IMAP enabled mi? (Gmail > Ayarlar > Yönlendirme ve POP/IMAP)", err=True) + sys.exit(1) + + # Save to .env (handle multi) + existing_emails = settings.gmail_emails + if existing_emails and email not in [e.strip() for e in existing_emails.split(",")]: + new_emails = existing_emails.strip() + f",{email}" + _upsert_env_file("GMAIL_EMAILS", new_emails) + existing_pw = settings.gmail_app_passwords or "" + # App passwords may contain spaces, so we join with comma + if existing_pw.strip(): + new_pw = existing_pw.strip() + f",{app_pw}" + else: + single_pw = settings.gmail_app_password or settings.gmail_password or "" + if single_pw and single_pw.strip() != app_pw: + new_pw = f"{single_pw.strip()},{app_pw}" + else: + new_pw = app_pw + _upsert_env_file("GMAIL_APP_PASSWORDS", new_pw) + click.echo(f"Wrote GMAIL_EMAILS and GMAIL_APP_PASSWORDS to .env (appended {email}, spaces preserved)") + else: + _upsert_env_file("GMAIL_EMAIL", email) + _upsert_env_file("GMAIL_APP_PASSWORD", app_pw) + click.echo("Wrote GMAIL_EMAIL and GMAIL_APP_PASSWORD to .env") + click.echo("ENABLE_CERT_TR=true olduğundan emin olun, sonra: app sync --source cert-tr") + return + + # OAuth path + # Need client_id/secret + client_id = settings.gmail_oauth_client_id + client_secret = settings.gmail_oauth_client_secret + if not client_id or not client_secret: + click.echo("Gmail OAuth için Google Cloud OAuth Client ID/Secret gerekli.") + click.echo("Oluştur: https://console.cloud.google.com/ -> APIs & Services -> Credentials -> Create Credentials -> OAuth Client ID (Desktop)") + click.echo("Redirect URI: http://localhost (InstalledAppFlow otomatik)") + click.echo("Scopes: https://mail.google.com/") + if not client_id: + client_id = click.prompt("OAuth Client ID", type=str, default=client_id or "").strip() + if not client_secret: + try: + client_secret = getpass.getpass("OAuth Client Secret (hidden): ").strip() + except (EOFError, KeyboardInterrupt): + click.echo("\nCancelled.", err=True) + sys.exit(1) + if not client_id or not client_secret: + click.echo("Client ID/Secret gerekli", err=True) + sys.exit(1) + # Save to .env + _upsert_env_file("GMAIL_OAUTH_CLIENT_ID", client_id) + _upsert_env_file("GMAIL_OAUTH_CLIENT_SECRET", client_secret) + click.echo("Wrote GMAIL_OAUTH_CLIENT_ID/SECRET to .env") + + # Run OAuth flow + click.echo(f"Tarayıcı açılacak, Gmail hesabı {email} ile onay verin...") + click.echo("Eğer tarayıcı açılmazsa, terminaldeki URL'yi manuel açın.") + try: + from advisory_rss.auth.gmail import run_gmail_oauth_flow, save_gmail_oauth_token + except ImportError as e: + click.echo(f"Gerekli paket eksik: {e}", err=True) + click.echo("Şunu çalıştır: pip install google-auth google-auth-oauthlib", err=True) + sys.exit(1) + + try: + refresh_token, access_token, expiry = run_gmail_oauth_flow(client_id, client_secret, email) + click.echo(f"+ OAuth başarılı: refresh_token alındı ({refresh_token[:8]}...)") + except (OSError, ValueError, RuntimeError) as e: + click.echo(f"- OAuth hatası: {_redact(str(e))}", err=True) + sys.exit(1) + + # Validate via XOAUTH2 IMAP + from advisory_rss.cert_tr.imap import connect_imap + + acc = { + "email": email, + "password": "", + "folder": settings.gmail_imap_folder or "INBOX", + "host": settings.gmail_imap_host or "imap.gmail.com", + "port": str(settings.gmail_imap_port), + "security": settings.gmail_imap_security or "SSL", + "auth_method": "oauth", + "oauth_refresh_token": refresh_token, + "oauth_client_id": client_id, + "oauth_client_secret": client_secret, + } + try: + mail = connect_imap(acc) + try: + mail.select(acc["folder"], readonly=True) + finally: + try: + mail.logout() + except (OSError, ValueError, RuntimeError) as e: + logger.debug("Gmail OAuth logout failed: %s", e) + click.echo(f"+ Gmail IMAP (OAuth) doğrulandı: {email}") + except (OSError, ValueError, RuntimeError) as e: + click.echo(f"- Gmail OAuth hatası: {_redact(str(e))}", err=True) + sys.exit(1) + + # Save refresh token + # Prefer file cache plus env + token_file = settings.gmail_oauth_token_file or "cache/gmail_oauth.json" + try: + save_gmail_oauth_token(token_file, email, refresh_token, client_id, client_secret, access_token, expiry) + click.echo(f"Wrote OAuth token to {token_file} (600 perms)") + except (OSError, ValueError, RuntimeError) as e: + logger.debug("Failed to save to file: %s", e) + + # Also save to .env for convenience (single or multi) + existing_emails = settings.gmail_emails + existing_rts = settings.gmail_oauth_refresh_tokens + if existing_emails and email not in [e.strip() for e in existing_emails.split(",")]: + new_emails = existing_emails.strip() + f",{email}" + _upsert_env_file("GMAIL_EMAILS", new_emails) + if existing_rts and existing_rts.strip(): + new_rts = existing_rts.strip() + f",{refresh_token}" + else: + single_rt = settings.gmail_oauth_refresh_token or "" + if single_rt and single_rt.strip() != refresh_token: + new_rts = f"{single_rt.strip()},{refresh_token}" + else: + new_rts = refresh_token + _upsert_env_file("GMAIL_OAUTH_REFRESH_TOKENS", new_rts) + click.echo(f"Wrote GMAIL_EMAILS and GMAIL_OAUTH_REFRESH_TOKENS to .env (appended {email})") + elif existing_emails: + # Update existing entry + # Find index and replace + emails = [e.strip() for e in existing_emails.split(",")] + rts = [r.strip() for r in (existing_rts or "").split(",")] if existing_rts else [] + # Pad rts + while len(rts) < len(emails): + rts.append("") + try: + idx = [e.lower() for e in emails].index(email.lower()) + rts[idx] = refresh_token + _upsert_env_file("GMAIL_OAUTH_REFRESH_TOKENS", ",".join(rts)) + click.echo("Updated GMAIL_OAUTH_REFRESH_TOKENS in .env") + except ValueError: + _upsert_env_file("GMAIL_OAUTH_REFRESH_TOKEN", refresh_token) + click.echo("Wrote GMAIL_OAUTH_REFRESH_TOKEN to .env") + else: + # Single + if settings.gmail_email and settings.gmail_email.strip().lower() != email.lower(): + # Migrate single to multi + prev_email = settings.gmail_email.strip() + prev_rt = settings.gmail_oauth_refresh_token or "" + _upsert_env_file("GMAIL_EMAILS", f"{prev_email},{email}") + _upsert_env_file("GMAIL_OAUTH_REFRESH_TOKENS", f"{prev_rt},{refresh_token}") + click.echo("Migrated single Gmail to multi (GMAIL_EMAILS)") + else: + _upsert_env_file("GMAIL_EMAIL", email) + _upsert_env_file("GMAIL_OAUTH_REFRESH_TOKEN", refresh_token) + click.echo("Wrote GMAIL_EMAIL and GMAIL_OAUTH_REFRESH_TOKEN to .env") + + click.echo("OAuth tamamlandı. ENABLE_CERT_TR=true olduğundan emin olun, sonra: app sync --source cert-tr") + click.echo("Not: Access token otomatik yenilenecek (refresh_token saklandı). İptal için Google Hesabı > Güvenlik > Üçüncü taraf erişimi'nden kaldırın.") + return - click.echo("") - click.echo("Token is VALID but NOT saved to disk by this command.") - click.echo("Set it in .env:") - click.echo(" echo 'GITHUB_TOKEN=YOUR_TOKEN_HERE' >> .env") - click.echo(" # or edit .env: GITHUB_TOKEN=github_pat_... / ghp_...") - click.echo("Then: app sync && app serve") - click.echo("Never commit .env (gitignored).") + click.echo(f"Bilinmeyen provider: {provider}", err=True) + sys.exit(1) @auth.command("status") diff --git a/src/advisory_rss/config/constants.py b/src/advisory_rss/config/constants.py index 7697144..6c1704a 100644 --- a/src/advisory_rss/config/constants.py +++ b/src/advisory_rss/config/constants.py @@ -42,6 +42,12 @@ DEFAULT_GMAIL_FOLDER = "INBOX" DEFAULT_CERT_TR_SENDER_ALLOWLIST = "siberguvenlik.gov.tr" +# Gmail OAuth +GMAIL_OAUTH_SCOPES = ["https://mail.google.com/"] +GMAIL_OAUTH_TOKEN_URI = "https://oauth2.googleapis.com/token" +GMAIL_OAUTH_AUTH_URI = "https://accounts.google.com/o/oauth2/auth" +DEFAULT_GMAIL_OAUTH_CACHE = "cache/gmail_oauth.json" + # Security TOKEN_REDACT_PATTERN = r"(gh[pousr]_[A-Za-z0-9_-]+|github_pat_[A-Za-z0-9_-]+)" diff --git a/src/advisory_rss/config/settings.py b/src/advisory_rss/config/settings.py index b6a60c7..0aa90d2 100644 --- a/src/advisory_rss/config/settings.py +++ b/src/advisory_rss/config/settings.py @@ -19,6 +19,7 @@ DEFAULT_GMAIL_IMAP_HOST, DEFAULT_GMAIL_IMAP_PORT, DEFAULT_GMAIL_IMAP_SECURITY, + DEFAULT_GMAIL_OAUTH_CACHE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL, DEFAULT_MAX_ITEMS, @@ -162,6 +163,22 @@ def __init__( gmail_imap_folders: str | None = Field( default=None, validation_alias="GMAIL_IMAP_FOLDERS" ) + # Gmail OAuth (alternative to App Password, per-account refresh token) + gmail_oauth_client_id: str | None = Field( + default=None, validation_alias="GMAIL_OAUTH_CLIENT_ID" + ) + gmail_oauth_client_secret: str | None = Field( + default=None, validation_alias="GMAIL_OAUTH_CLIENT_SECRET" + ) + gmail_oauth_refresh_token: str | None = Field( + default=None, validation_alias="GMAIL_OAUTH_REFRESH_TOKEN" + ) + gmail_oauth_refresh_tokens: str | None = Field( + default=None, validation_alias="GMAIL_OAUTH_REFRESH_TOKENS" + ) + gmail_oauth_token_file: str = Field( + default=DEFAULT_GMAIL_OAUTH_CACHE, validation_alias="GMAIL_OAUTH_TOKEN_FILE" + ) # Optional: per-account JSON-like override not needed; plural fields suffice cert_tr_sender_allowlist: str = Field( default=DEFAULT_CERT_TR_SENDER_ALLOWLIST, @@ -428,6 +445,19 @@ def _split_list(self, raw: str | None) -> list[str]: parts = re.split(r"[,\s;]+", raw.strip()) return [p.strip() for p in parts if p.strip()] + def _split_folders(self, raw: str | None) -> list[str]: + if not raw or not raw.strip(): + return [] + import re + + # Split only on comma/semicolon to preserve spaces inside folder names (e.g., "CVE Yazışmaları") + parts = [p.strip() for p in re.split(r"[;,]+", raw) if p.strip()] + if len(parts) <= 1 and "," not in raw and ";" not in raw: + # Fallback for backward compat where folders were space-separated without comma + # Only if no comma/semicolon present, split on whitespace/comma/semicolon + parts = [p.strip() for p in re.split(r"[,\s;]+", raw.strip()) if p.strip()] + return parts + def get_proton_accounts(self) -> list[dict[str, str]]: """Resolve multi-account Proton config. @@ -449,14 +479,14 @@ def get_proton_accounts(self) -> list[dict[str, str]]: if self.proton_bridge_passwords: passwords = self._split_list(self.proton_bridge_passwords) if self.proton_imap_folders: - folders = self._split_list(self.proton_imap_folders) + folders = self._split_folders(self.proton_imap_folders) elif self.proton_bridge_email and self.proton_bridge_email.strip(): emails = [self.proton_bridge_email.strip()] if self.proton_bridge_password: passwords = [self.proton_bridge_password.strip()] # folder for single: use explicit plural if set else singular if self.proton_imap_folders and self.proton_imap_folders.strip(): - folders = self._split_list(self.proton_imap_folders) + folders = self._split_folders(self.proton_imap_folders) else: folders = [self.proton_imap_folder.strip() or DEFAULT_PROTON_FOLDER] else: @@ -468,6 +498,23 @@ def get_proton_accounts(self) -> list[dict[str, str]]: # If single password provided but multiple emails, reuse it if len(passwords) == 1 and len(emails) > 1: passwords = passwords * len(emails) + # Handle single email with multiple folders: create one account per folder + if len(emails) == 1 and len(folders) > 1: + single_email = emails[0] + single_pw = passwords[0] if passwords else (self.proton_bridge_password or "") + for folder in folders: + folder = folder.strip() or default_folder + out.append( + { + "email": single_email.strip(), + "password": (single_pw or "").strip(), + "folder": folder, + "host": (self.proton_bridge_host or DEFAULT_PROTON_IMAP_HOST).strip(), + "port": str(self.proton_imap_port), + "security": (self.proton_imap_security or DEFAULT_PROTON_IMAP_SECURITY).strip().upper(), + } + ) + return out for idx, email in enumerate(emails): if not email or "@" not in email: continue @@ -495,53 +542,162 @@ def get_proton_accounts(self) -> list[dict[str, str]]: ) return out + def _load_gmail_oauth_file_tokens(self) -> dict[str, object]: + """Load Gmail OAuth tokens from JSON file (cache/gmail_oauth.json). + + File format: {"client_id": "...", "client_secret": "...", "accounts": {"a@gmail.com": {"refresh_token": "..."}}} + or flat: {"a@gmail.com": "refresh_token", ...} + Returns dict with keys possibly containing 'accounts' or direct email->token. + """ + import json + import logging + from pathlib import Path + try: + path = Path(self.gmail_oauth_token_file) + if not path.exists(): + return {} + data = json.loads(path.read_text(encoding="utf-8")) + return data if isinstance(data, dict) else {} + except (OSError, ValueError, json.JSONDecodeError) as e: + logging.getLogger(__name__).debug("Failed to load Gmail OAuth file %s: %s", self.gmail_oauth_token_file, e) + return {} + def get_gmail_accounts(self) -> list[dict[str, str]]: """Resolve Gmail IMAP accounts for CERT-TR. - Supports single (GMAIL_EMAIL + GMAIL_APP_PASSWORD) and - multi (GMAIL_EMAILS, GMAIL_APP_PASSWORDS comma-separated, - GMAIL_IMAP_FOLDERS). Host/port/security are shared per Gmail - config (defaults imap.gmail.com:993 SSL). - Each account is {email,password,folder,host,port,security,provider}. + Supports: + - App Password: single (GMAIL_EMAIL + GMAIL_APP_PASSWORD) and + multi (GMAIL_EMAILS, GMAIL_APP_PASSWORDS comma-separated, + GMAIL_IMAP_FOLDERS). Host/port/security are shared per Gmail + config (defaults imap.gmail.com:993 SSL). + - OAuth: GMAIL_OAUTH_CLIENT_ID/SECRET + GMAIL_OAUTH_REFRESH_TOKEN(S) + or cache/gmail_oauth.json. For each email, if a refresh token + exists (env or file), the account will use XOAUTH2 (auth_method=oath). + Otherwise App Password is used. + Each account is {email,password,folder,host,port,security,provider,auth_method, + oauth_client_id,oauth_client_secret,oauth_refresh_token}. """ emails: list[str] = [] passwords: list[str] = [] folders: list[str] = [] + oauth_refresh_tokens: list[str] = [] - # Helper to split passwords preserving internal spaces (comma-only) def _split_passwords(raw: str | None) -> list[str]: if not raw or not raw.strip(): return [] - # split only on comma, keep internal spaces parts = [p.strip() for p in raw.split(",")] return [p for p in parts if p] + # Determine emails source if self.gmail_emails and self.gmail_emails.strip(): emails = self._split_list(self.gmail_emails) if self.gmail_app_passwords: passwords = _split_passwords(self.gmail_app_passwords) - elif self.gmail_app_password or self.gmail_password: - # single fallback handled later - pass + if self.gmail_oauth_refresh_tokens: + oauth_refresh_tokens = _split_passwords(self.gmail_oauth_refresh_tokens) + elif self.gmail_oauth_refresh_token: + oauth_refresh_tokens = [self.gmail_oauth_refresh_token.strip()] if self.gmail_imap_folders: - folders = self._split_list(self.gmail_imap_folders) + folders = self._split_folders(self.gmail_imap_folders) elif self.gmail_email and self.gmail_email.strip(): emails = [self.gmail_email.strip()] - # Prefer app password, fallback to GMAIL_PASSWORD alias pw = self.gmail_app_password or self.gmail_password if pw: passwords = [pw.strip()] + # OAuth single + if self.gmail_oauth_refresh_tokens and self.gmail_oauth_refresh_tokens.strip(): + oauth_refresh_tokens = _split_passwords(self.gmail_oauth_refresh_tokens) + elif self.gmail_oauth_refresh_token: + oauth_refresh_tokens = [self.gmail_oauth_refresh_token.strip()] if self.gmail_imap_folders and self.gmail_imap_folders.strip(): - folders = self._split_list(self.gmail_imap_folders) + folders = self._split_folders(self.gmail_imap_folders) else: folders = [self.gmail_imap_folder.strip() or DEFAULT_GMAIL_FOLDER] else: - return [] + # Check OAuth file for accounts even if no env email + file_tokens = self._load_gmail_oauth_file_tokens() + # file may contain {"accounts": {"a@gmail.com": {"refresh_token": "..."}}} + accounts_map: dict[str, str] = {} + if isinstance(file_tokens, dict): + if "accounts" in file_tokens and isinstance(file_tokens["accounts"], dict): + for em, info in file_tokens["accounts"].items(): + if isinstance(info, dict) and info.get("refresh_token"): + accounts_map[em] = str(info["refresh_token"]) + elif isinstance(info, str): + accounts_map[em] = info + else: + # flat email->token + for k, v in file_tokens.items(): + if "@" in k and isinstance(v, str): + accounts_map[k] = v + elif "@" in k and isinstance(v, dict) and v.get("refresh_token"): + accounts_map[k] = str(v["refresh_token"]) + if accounts_map: + emails = list(accounts_map.keys()) + oauth_refresh_tokens = [accounts_map[e] for e in emails] + # folders will be default + else: + return [] + # Also check file for additional OAuth tokens that may supplement env + file_tokens = self._load_gmail_oauth_file_tokens() + file_map: dict[str, str] = {} + if isinstance(file_tokens, dict): + if "accounts" in file_tokens and isinstance(file_tokens["accounts"], dict): + for em, info in file_tokens["accounts"].items(): + if isinstance(info, dict) and info.get("refresh_token"): + file_map[em.lower()] = str(info["refresh_token"]) + elif isinstance(info, str): + file_map[em.lower()] = info + else: + for k, v in file_tokens.items(): + if "@" in k: + if isinstance(v, str): + file_map[k.lower()] = v + elif isinstance(v, dict) and v.get("refresh_token"): + file_map[k.lower()] = str(v["refresh_token"]) + # Also consider env file client_id/secret for file-based accounts out: list[dict[str, str]] = [] default_folder = (self.gmail_imap_folder or DEFAULT_GMAIL_FOLDER).strip() or DEFAULT_GMAIL_FOLDER if len(passwords) == 1 and len(emails) > 1: passwords = passwords * len(emails) + if len(oauth_refresh_tokens) == 1 and len(emails) > 1: + oauth_refresh_tokens = oauth_refresh_tokens * len(emails) + # Handle single email with multiple folders: expand to multiple accounts + if len(emails) == 1 and len(folders) > 1: + single_email = emails[0] + single_pw = passwords[0] if passwords else (self.gmail_app_password or self.gmail_password or "") + single_oauth = oauth_refresh_tokens[0] if oauth_refresh_tokens else "" + for folder in folders: + folder = folder.strip() or default_folder + # Determine OAuth token for this folder (same for all folders of same email) + oauth_rt = single_oauth + # Also check file_map for this email + # file_map will be computed later, but for now handle simple + auth_method = "oauth" if oauth_rt else "password" + acct: dict[str, str] = { + "email": single_email.strip(), + "password": (single_pw or "").strip(), + "folder": folder, + "host": (self.gmail_imap_host or DEFAULT_GMAIL_IMAP_HOST).strip(), + "port": str(self.gmail_imap_port), + "security": (self.gmail_imap_security or DEFAULT_GMAIL_IMAP_SECURITY).strip().upper(), + "provider": "gmail", + "auth_method": auth_method, + } + if oauth_rt: + acct["oauth_refresh_token"] = oauth_rt.strip() + cid = self.gmail_oauth_client_id or "" + csec = self.gmail_oauth_client_secret or "" + if not cid and isinstance(file_tokens, dict) and file_tokens.get("client_id"): + cid = str(file_tokens["client_id"]) + if not csec and isinstance(file_tokens, dict) and file_tokens.get("client_secret"): + csec = str(file_tokens["client_secret"]) + acct["oauth_client_id"] = (cid or "").strip() + acct["oauth_client_secret"] = (csec or "").strip() + acct["oauth_token_file"] = (self.gmail_oauth_token_file or "").strip() + out.append(acct) + return out for idx, email in enumerate(emails): if not email or "@" not in email: continue @@ -551,24 +707,47 @@ def _split_passwords(raw: str | None) -> list[str]: elif passwords: pw = passwords[0] else: - # fallback single single_pw = self.gmail_app_password or self.gmail_password pw = single_pw or "" + # Determine OAuth token for this email (env takes precedence, then file) + oauth_rt = "" + if idx < len(oauth_refresh_tokens) and oauth_refresh_tokens[idx]: + oauth_rt = oauth_refresh_tokens[idx] + elif file_map.get(email.lower()): + oauth_rt = file_map[email.lower()] + elif self.gmail_oauth_refresh_token and len(emails) == 1: + oauth_rt = self.gmail_oauth_refresh_token folder = default_folder if idx < len(folders) and folders[idx]: folder = folders[idx] folder = folder.strip() or default_folder - out.append( - { - "email": email.strip(), - "password": pw.strip() if pw else "", - "folder": folder, - "host": (self.gmail_imap_host or DEFAULT_GMAIL_IMAP_HOST).strip(), - "port": str(self.gmail_imap_port), - "security": (self.gmail_imap_security or DEFAULT_GMAIL_IMAP_SECURITY).strip().upper(), - "provider": "gmail", - } - ) + # Decide auth method: prefer OAuth if refresh token present + auth_method = "oauth" if oauth_rt else "password" + # For OAuth, password is not used, but keep for fallback + acct: dict[str, str] = { + "email": email.strip(), + "password": pw.strip() if pw else "", + "folder": folder, + "host": (self.gmail_imap_host or DEFAULT_GMAIL_IMAP_HOST).strip(), + "port": str(self.gmail_imap_port), + "security": (self.gmail_imap_security or DEFAULT_GMAIL_IMAP_SECURITY).strip().upper(), + "provider": "gmail", + "auth_method": auth_method, + } + if oauth_rt: + acct["oauth_refresh_token"] = oauth_rt.strip() + # client_id/secret may be in env or file + cid = self.gmail_oauth_client_id or "" + csec = self.gmail_oauth_client_secret or "" + # also check file for client_id/secret + if not cid and isinstance(file_tokens, dict) and file_tokens.get("client_id"): + cid = str(file_tokens["client_id"]) + if not csec and isinstance(file_tokens, dict) and file_tokens.get("client_secret"): + csec = str(file_tokens["client_secret"]) + acct["oauth_client_id"] = (cid or "").strip() + acct["oauth_client_secret"] = (csec or "").strip() + acct["oauth_token_file"] = (self.gmail_oauth_token_file or "").strip() + out.append(acct) return out def get_cert_tr_accounts(self) -> list[dict[str, str]]: From b39ea308fa12435452d55fde58113eb4f8def101 Mon Sep 17 00:00:00 2001 From: Lunixizm0 <216941080+Lunixizm0@users.noreply.github.com> Date: Sun, 20 Sep 2026 21:11:28 +0300 Subject: [PATCH 4/4] ruff formatting and readme.md update --- README.md | 276 +++++++++++++++++++-------- src/advisory_rss/auth/gmail.py | 96 +++++----- src/advisory_rss/cache/models.py | 2 - src/advisory_rss/cache/store.py | 21 +- src/advisory_rss/cert_tr/__init__.py | 2 +- src/advisory_rss/cert_tr/imap.py | 47 ++--- src/advisory_rss/cert_tr/parser.py | 22 +-- src/advisory_rss/cert_tr/source.py | 12 +- src/advisory_rss/cli.py | 120 ++++++++---- src/advisory_rss/config/settings.py | 73 ++++--- src/advisory_rss/logging_config.py | 1 + src/advisory_rss/rss/builder.py | 14 +- src/advisory_rss/server/app.py | 35 +++- tests/unit/test_cert_tr_imap.py | 28 ++- tests/unit/test_cert_tr_parser.py | 8 +- tests/unit/test_cert_tr_rss.py | 33 +++- tests/unit/test_cert_tr_settings.py | 19 +- tests/unit/test_normalize.py | 2 +- 18 files changed, 541 insertions(+), 270 deletions(-) diff --git a/README.md b/README.md index c92c321..0e3cc4e 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,36 @@ -# Advisory-RSS - GitHub Security Advisories - Local RSS +# Advisory-RSS - GitHub + CERT-TR (SGB) -Security-first local service that lists **all GitHub Security Advisories created/owned by the authenticated account** and exposes them as an **RSS 2.0 feed** for Zen Browser's RSS Live Folder. -What the hell is 'Zen Browser's Live Folder?' == `https://github.com/zen-browser/desktop/releases/tag/1.19b` +Security-first local service that lists **GitHub Security Advisories created/owned by you** **and** **CERT-TR / T.C. Siber Güvenlik Başkanlığı advisories received via e-mail (Proton Bridge or Gmail)** and exposes them as a **single mixed RSS 2.0 feed** for Zen Browser's RSS Live Folder. What is 'Zen Browser's Live Folder?' == `https://github.com/zen-browser/desktop/releases/tag/1.19b` -GitHub API - authenticated PAT - fetch advisories owned/created by account - normalize + cache - RSS 2.0 on 127.0.0.1 - Zen Browser's Live Folder +```mermaid +flowchart LR + GH["GitHub API
(PAT)"] + PB["Proton Bridge IMAP
siberguvenlik.gov.tr
BODY.PEEK — never marks read"] + GM["Gmail IMAP
imap.gmail.com
App Password / XOAUTH2
never marks read"] + + GH --> NORM["normalize + cache
(advisories.db)"] + PB --> NORM + GM --> NORM + + NORM --> RSS["RSS 2.0
on 127.0.0.1"] + RSS --> ZEN["Zen Live Folder"] + + style GH fill:#24292f,stroke:#1f2328,color:#fff + style PB fill:#6d28d9,stroke:#4c1d95,color:#fff + style GM fill:#dc2626,stroke:#991b1b,color:#fff + style NORM fill:#f59e0b,stroke:#92400e,color:#000 + style RSS fill:#0ea5e9,stroke:#0c4a6e,color:#fff + style ZEN fill:#059669,stroke:#064e3b,color:#fff +``` **Endpoints:** ``` -http://127.0.0.1:PORT/rss.xml -http://127.0.0.1:PORT/health +http://127.0.0.1:PORT/rss.xml # mixed (GitHub + CERT-TR) +http://127.0.0.1:PORT/rss.xml?source=github +http://127.0.0.1:PORT/rss.xml?source=cert-tr +http://127.0.0.1:PORT/health # per-source counts, cert_tr diag +http://127.0.0.1:PORT/ # hint ``` --- @@ -18,32 +39,49 @@ http://127.0.0.1:PORT/health - **Python 3.12+** - `uv` **or** `pip` + venv -- A GitHub account that **owns/creates advisories** (i.e., you are `author` of advisories) -- A GitHub PAT (see below) +- For GitHub source: a GitHub account that **owns/creates advisories** (you are `author`) + PAT (see §3) +- For CERT-TR source (optional, `ENABLE_CERT_TR=true`): + - **Proton Mail** with **Bridge** installed (paid plan required, you can patch the source though) **or** **Gmail** with IMAP enabled + - CERT-TR mails come from `cve@siberguvenlik.gov.tr` / `siberguvenlik.gov.tr` - only **CVE-assigned** mails are ingested (like `CVE ID Talebi`, `Zafiyet Bildiriminiz Hakkında` without CVE are skipped) + - Proton: Bridge exposes localhost IMAP `127.0.0.1:1143` `STARTTLS` (self-signed) + - Gmail: `imap.gmail.com:993` `SSL` via **App Password** (16 chars, 2FA required) **or** **OAuth 2.0** (`google-auth` + `google-auth-oauthlib`) --- ## 2. Installation +### 2.1 User Install + +```bash +pip install advisory-rss +pipx install advisory-rss +``` + +### 2.2 Dev Install + ```bash git clone https://github.com/Lunixizm0/advisory-rss cd advisory-rss # with uv (recommended) -uv sync +uv sync --extra dev # or with pip python -m venv .venv && source .venv/bin/activate pip install -e .[dev] # verify app --help -pytest -q +app auth login --help +pytest -q # 78 tests +ruff check && pyright # All checks passed! ``` --- ## 3. GitHub PAT -The service uses **official GitHub REST API** + +The service uses **official GitHub REST API** for the GitHub source. + | PAT type | Where to create | Scopes / Permissions required | When needed | |---|---|---|---| | **Fine-grained PAT (recommended)** | | Resource owner: *your user + relevant orgs* - Repository access: *All repositories* (or select repos) - **Repository permissions - Security advisories: Read-only** + **Metadata: Read-only** (implied). For org-wide bulk fetch: **Organization permissions - Security advisories: Read-only** (requires org owner/security-manager). | Preferred - short expiry (30–90 days), no read to code. | @@ -64,27 +102,25 @@ The service uses **official GitHub REST API** --- -## 4. First Sync +## 4. CERT-TR / Gmail / Proton + +### 4a. GitHub only (default) ```bash cp .env.example .env -# edit .env - set at least GITHUB_TOKEN $EDITOR .env # e.g. GITHUB_TOKEN=github_pat_… or ghp_… # leave BIND_ADDRESS=127.0.0.1 and PORT=8765 - do NOT change to 0.0.0.0 # validate PAT (only checks GET /user, never saves to file - .env is the only source) -app auth login -# Paste PAT -> "+ Validated as @you" -> then set GITHUB_TOKEN in .env as instructed +app auth login --provider github +# Paste PAT -> "+ Validated as @you" -> choose to write to .env or set manually # manual fetch - writes cache/advisories.db app sync - -# inspect +app sync --source github # only GitHub app status -# - Authenticated user: @octocat -# - Cached advisories: 12 -# - RSS URL: http://127.0.0.1:8765/rss.xml +# - GitHub: 1 CERT-TR: 2 (if CERT-TR enabled) ``` **Direct fetch for advisories in repos you don't own** (you authored GHSA in `other-org/other-repo` but `affiliation=owner` doesn't list it - GitHub has no `author` filter, see Known Limitations): @@ -103,7 +139,47 @@ LOG_LEVEL=INFO app sync - `author_or_publisher` - also matches `publisher.login` - `author_or_collaborator` - also matches `collaborating_users[]` (covers advisories where you are collaborator but not author) -Do not equate "advisories in repos I own" with "created by me" - they are different. +### 4b. CERT-TR via Proton Bridge + +Bridge must be installed, logged in, and running (`protonmail-bridge --cli` - `info` shows host/port/password). + +```bash +cp .env.example .env +$EDITOR .env + +# interactive login (validates via IMAP SELECT, does not mark read) +app auth login --provider proton --email mail@mail.com +# Paste Bridge password -> "+ Proton IMAP doğrulandı" +# Writes PROTON_BRIDGE_EMAIL/PASSWORD (or appends to PROTON_BRIDGE_EMAILS) to .env + +app sync --source cert-tr # only CERT-TR +app sync --source all # GitHub + CERT-TR mixed +app status # shows CERT-TR accounts: [proton] mail@mail.com -> folder=INBOX host=127.0.0.1:1143 +``` + +Bridge's IMAP is `127.0.0.1:1143` `STARTTLS` with self-signed cert (`CERT_NONE`), `BODY.PEEK` + `SELECT readonly=True` (never sets `\Seen`), reconnect every 25 mails (Bridge drops). + +### 4c. CERT-TR via Gmail (App Password or OAuth) + +**App Password (simplest, 2FA required):** + +1. Gmail - Google Hesabı - Güvenlik - 2 Adımlı Doğrulama - Uygulama şifreleri - `Posta` - `Diğer` - `Advisory-RSS` - 16 haneli şifre (örn. `abcd efgh ijkl mnop`, boşluklar korunur). +2. Gmail - Ayarlar - Yönlendirme ve POP/IMAP - IMAP'ı etkinleştir. + +```bash +app auth login --provider gmail --method app-password --email mail@mail.com +# Paste App Password -> "+ Gmail IMAP (App Password) doğrulandı" +``` + +**OAuth 2.0 (recommended for long-lived, no app password):** + +1. - Create Project - APIs & Services - Credentials - Create Credentials - OAuth Client ID - Application type: **Desktop** - Name: `Advisory-RSS` - Create - copy `Client ID` / `Client Secret`. +2. Enable **Gmail API**: APIs & Services - Enable APIs - search `Gmail API` - Enable. +3. The CLI will open browser for consent (`https://mail.google.com/` scope, `access_type=offline` `prompt=consent` to get `refresh_token`). + +```bash +app auth login --provider gmail --method oauth --email mail@mail.com +``` --- @@ -111,30 +187,31 @@ Do not equate "advisories in repos I own" with "created by me" - they are differ ```bash app serve -# binds 127.0.0.1:8765, background refresh every REFRESH_INTERVAL (default 600s = 10 min) -# health: curl http://127.0.0.1:8765/health +# binds 127.0.0.1:8765, background refresh every REFRESH_INTERVAL (default 600s = 10 min) for BOTH sources +# health: curl http://127.0.0.1:8765/health | jq # feed : curl http://127.0.0.1:8765/rss.xml | xmllint --noout - +# filter: curl "http://127.0.0.1:8765/rss.xml?source=cert-tr" | xmllint --noout - ``` ## 6. RSS URL -Canonical for Zen Browser: +Canonical for Zen Browser (mixed): ``` http://127.0.0.1:8765/rss.xml +http://127.0.0.1:8765/rss.xml?source=github +http://127.0.0.1:8765/rss.xml?source=cert-tr ``` -Do not use HTTPS. Also, no auth on `/rss.xml` (Zen has no RSS auth) - ---- +Do not use HTTPS. Also, no auth on `/rss.xml` (Zen has no RSS auth). Mixed feed title: `Security Advisories (GitHub + CERT-TR) - @you`, per-source: `[CERT-TR] pardus-etap-settings: CVE-2026-92083 (CWE-862)`. ## 7. Adding the Feed to Zen Browser RSS Dynamic Folder 1. Start the feed: `app serve` and verify `curl http://127.0.0.1:8765/rss.xml | head` returns ``. 2. Open Zen Browser - **Right Click - Live Folder - Add RSS Feed** 3. Paste **`http://127.0.0.1:8765/rss.xml`** as the feed source. Leave protocol HTTP (not HTTPS). -4. Save. Advisories appear sorted `updated_at DESC`, newest first. Withdrawn advisories remain with `[WITHDRAWN]` prefix - not deleted. -5. If Zen shows empty folder: `curl http://127.0.0.1:8765/health | jq .advisories_count` should be >0; if 0, re-run `app sync` and check `app status` for rate-limit/auth errors. +4. Save. Advisories appear sorted `updated_at DESC`, newest first. Withdrawn advisories remain with `[WITHDRAWN]` prefix - not deleted. CERT-TR items have `[CERT-TR]` prefix, NVD link, `source:cert-tr`/`cve:`/`cwe:` categories. +5. If Zen shows empty folder: `curl http://127.0.0.1:8765/health | jq .advisories_count,.github_count,.cert_tr_count` should be >0; if 0, re-run `app sync` and check `app status` for rate-limit/auth errors. --- @@ -147,7 +224,7 @@ REFRESH_INTERVAL=300 # 5 minutes, min 60 MAX_ITEMS=500 # RSS truncation (cache still retains all to avoid missing advisories) ``` -Restart `app serve` to apply. The feed `Cache-Control: private, max-age=300, must-revalidate` and `5` update automatically. `app status` shows `Next scheduled sync`. +Restart `app serve` to apply. The feed `Cache-Control: private, max-age=300, must-revalidate` and `5` update automatically. `app status` shows `Next scheduled sync`. `background_refresh_loop` `src/advisory_rss/server/app.py:346` polls both GitHub (`If-None-Match`/`ETag`) and CERT-TR (`IMAP` `BODY.PEEK`) every interval, `asyncio.to_thread` for IMAP, stale kept on error. ## 8b. Keep running after closing the terminal (`--daemon`) @@ -156,7 +233,7 @@ Normal `app serve` dies when the terminal closes (SIGHUP). If you want it to kee ```bash # easiest - daemonize (double-fork + setsid, like nohup but with pid/log management) app serve --daemon -# → pid: cache/advisory-rss.pid , log: cache/serve.log +# - pid: cache/advisory-rss.pid , log: cache/serve.log # verification cat cache/advisory-rss.pid ps -p $(cat cache/advisory-rss.pid) -o pid,cmd @@ -179,7 +256,7 @@ tmux new -s rss "app serve" # systemctl --user daemon-reload && systemctl --user enable --now advisory-rss ``` -Background behaviour: RSS reads cache only; a background task polls GitHub every `REFRESH_INTERVAL`, uses `If-None-Match`/`ETag` conditional requests, and **keeps stale cache on transient GitHub errors** (requirement: feed stays available if GitHub is down). +Background behaviour: RSS reads cache only; a background task polls **both** sources every `REFRESH_INTERVAL`, uses `If-None-Match`/`ETag` for GitHub and `BODY.PEEK`/`readonly=True` for IMAP, and **keeps stale cache on transient errors** (requirement: feed stays available if GitHub/Gmail/Proton is down). --- @@ -187,96 +264,139 @@ Background behaviour: RSS reads cache only; a background task polls GitHub every | Symptom | Check | |---|---| -| `401 Unauthorized` at `app sync` | PAT expired/revoked or empty. Re-run `app auth login`; verify at . See Health `last_error`. | +| `401 Unauthorized` at `app sync` | PAT expired/revoked or empty. Re-run `app auth login --provider github`; verify at . See Health `last_error`. | | `403 Forbidden` | PAT scopes missing `repository_advisories:read` or org role insufficient (org bulk needs owner/security-manager - falls back to per-repo). Log shows scope hint. | | `429 / rate limited` | `health.rate_limited_until` set, `app status` shows it. Wait or raise `MAX_REPOS`/reduce `REFRESH_INTERVAL` frequency. Logs include `Retry-After`/`X-RateLimit-Reset`. | | `404` for a repo | Repo not found or no advisories - skipped, not fatal. | | `422 Validation` | GitHub rejected params (e.g., bad state) - logged and skipped. | -| Feed empty but advisories exist | Check `FILTER_MODE`. Default `author` requires you are `author`. Try `FILTER_MODE=author_or_publisher`. Also ensure `authenticated_user` in `app status` equals your advisory author. | +| Feed empty but advisories exist | Check `FILTER_MODE` (default `author` requires you are `author`. Try `author_or_publisher`). Also ensure `authenticated_user` in `app status` equals your advisory author. For CERT-TR: check `ENABLE_CERT_TR=true` and `app status` - `CERT-TR accounts: N` and `IMAP search empty` - folder name wrong? Try `GMAIL_IMAP_FOLDERS=INBOX` only, or check `LOG_LEVEL=DEBUG app sync`. | +| `IMAP login failed` (Proton) | Bridge running? `127.0.0.1:1143` reachable? `protonmail-bridge --cli` - `info` shows correct host/port/password. Password is **Bridge password**, not account password. Check `PROTON_BRIDGE_HOST/PORT/SECURITY`. | +| `IMAP login failed` (Gmail App Password) | 2FA enabled? App Passwords - generate new 16-char (spaces preserved). Gmail - Settings - Forwarding and POP/IMAP - IMAP enabled. Check `GMAIL_IMAP_HOST=imap.gmail.com:993 SSL`. | +| `IMAP XOAUTH2 failed` (Gmail OAuth) | `GMAIL_OAUTH_CLIENT_ID/SECRET` correct? Refresh token expired/revoked? Re-run `app auth login --provider gmail --method oauth`. Check `cache/gmail_oauth.json` exists (`chmod 600`). Ensure Gmail API enabled in Google Cloud. Check `LOG_LEVEL=DEBUG` for `fetch_access_token` error. `https://oauth2.googleapis.com/token` must be reachable. | +| `UnicodeEncodeError: ascii ... Yazışmaları` | Fixed in `src/advisory_rss/cert_tr/imap.py:24` via `modified UTF-7` fallback + `mail._encoding='utf-8'` + `ENABLE UTF8=ACCEPT`. Ensure `GMAIL_IMAP_FOLDERS` uses `,` as separator (not `;` + space split) and folder name exactly as in Gmail UI (case-sensitive). Try `LOG_LEVEL=DEBUG app sync --source cert-tr` to see `IMAP select folder`. | +| `CERT-TR sync: no advisories (raw 0)` | `CERT_TR_SENDER_ALLOWLIST=siberguvenlik.gov.tr` correct? Check `IMAP search empty` - folder empty or `FROM` mismatch. Try `GMAIL_IMAP_FOLDERS=INBOX` and check Gmail for mails from `cve@siberguvenlik.gov.tr`. `CERT_TR_MAX_MAILS` maybe too low? `CERT_TR_SEARCH_DAYS` maybe filtering? | +| `CERT-TR sync OK - 0 advisories ... parsed 0` but raw >0 | All mails without CVE are now **skipped** (CVE-only). Check `health` - `cert_tr_count` should be 2 for your `CVE-2026-92083/16245`. Old `CERT-TR-MSG-*` rows auto-deleted on next `CacheStore` init `src/advisory_rss/cache/store.py:76`. | | `pip-audit` failure | `pip install pip-audit && pip-audit` - fix by bumping pinned dep in `pyproject.toml`. | +| `google-auth-oauthlib` missing | `uv sync --extra dev` or `pip install google-auth google-auth-oauthlib` (added to `pyproject.toml:9`). | View logs: `LOG_LEVEL=DEBUG app serve` for detail (still redacted). -Verify feed: `xmllint --noout <(curl -s http://127.0.0.1:8765/rss.xml)` - no output means valid. +Verify feed: `xmllint --noout <(curl -s http://127.0.0.1:8765/rss.xml)` - no output means valid. Check per-source: `curl -s "http://127.0.0.1:8765/rss.xml?source=cert-tr" | grep CERT-TR`. --- ## Architecture Explanation -``` -GitHub REST API (api.github.com) - │ - │ PAT (Authorization: Bearer, X-GitHub-Api-Version: 2022-11-28) - ▼ -get_user - list_user_orgs - list_user_repos (paginated, Link:max 100/page) [skipped if SKIP_FULL_SCAN=true] - │ │ - │ └─► orgs/{org}/security-advisories?state=… (bulk where authorized) - └─► repos/{owner}/{repo}/security-advisories?state=…&after=… (per-repo, per-state) - ▲ plus GITHUB_REPOS/GITHUB_ORG direct fetch (always scanned, even if SKIP_FULL_SCAN) - │ handle 401/403/404/422/429/5xx, ETag/If-None-Match, 304 - ▼ - normalize (github/normalize.py - tolerant, never crashes on one bad advisory) - │ - filter by author (github/filter.py - author | author_or_publisher | author_or_collaborator) - │ - dedup by GHSA, sort updated_at DESC - │ - upsert - cache/advisories.db (SQLite WAL, parameterized, ETag map, cache fallback) - │ - background_refresh_loop (asyncio every REFRESH_INTERVAL, stale kept on GitHub outage) - │ - RSS request (no GitHub call) ◄┘ - │ - ▼ - GET http://127.0.0.1:8765/rss.xml - builder (xml_util escaping, RFC822 pubDate, stable GHSA GUID, withdrawn kept) - │ - ▼ - Zen Browser RSS Dynamic Folder +```mermaid +flowchart TD + subgraph GH["GitHub REST API — api.github.com"] + direction TB + GH_AUTH["PAT Bearer
2022-11-28"] + GH_A1["get_user
list_user_orgs
list_user_repos"] + GH_A2["orgs/{org}/security-advisories
(bulk, owner/security-manager)"] + GH_A3["repos/{owner}/{repo}/security-advisories
per_page=100, paginated via Link header
+ GITHUB_REPOS / GITHUB_ORG"] + GH_A4["HTTP handling
401/403/404/422/429/5xx
ETag / If-None-Match → 304"] + GH_A5["normalize
filter by author
FILTER_MODE: author / author_or_publisher / collaborator"] + GH_A6["dedup by GHSA
sort updated_at DESC"] + + GH_AUTH --> GH_A1 + GH_A1 --> GH_A2 + GH_A1 --> GH_A3 + GH_A2 --> GH_A4 + GH_A3 --> GH_A4 + GH_A4 --> GH_A5 --> GH_A6 + end + + subgraph CERT["CERT-TR IMAP
Proton 127.0.0.1:1143 / Gmail imap.gmail.com:993"] + direction TB + CT_AUTH["OAuth XOAUTH2
refresh_token → access_token
or App Password / Bridge"] + CT_B1["connect_imap() per account
{email, host, port, security, auth_method}"] + CT_B2["_ssl_context_for_host()
CERT_NONE: Bridge / CERT_REQUIRED: Gmail"] + CT_B3["_select_folder_imap()
UTF-8 + ENABLE UTF8=ACCEPT
modified UTF-7 fallback — CVE Yazışmaları"] + CT_B4["IMAP SEARCH
FROM siberguvenlik.gov.tr [SINCE ...]
per folder — readonly + BODY.PEEK"] + CT_B5["fetch BODY.PEEK[]
batch 25 + reconnect"] + CT_B6["parse_cert_tr_email()
src/advisory_rss/cert_tr/parser.py:285
strict CVE-only, CWE / product / NVD refs"] + CT_B7["is_cert_tr_sender()
allowlist check"] + CT_B8["dedup by CERT-TR-CVE-...
sort"] + + CT_AUTH --> CT_B1 --> CT_B2 --> CT_B3 --> CT_B4 --> CT_B5 --> CT_B6 --> CT_B7 --> CT_B8 + end + + AGG["aggregator
mixed — no collision GHSA-* vs CERT-TR-*"] + CACHE["upsert — cache/advisories.db
SQLite WAL, source column, ETag map, MSG cleanup"] + LOOP["background_refresh_loop
asyncio every REFRESH_INTERVAL
stale kept on error"] + REQ["RSS request
cache only — ?source= filter"] + FEED["GET http://127.0.0.1:8765/rss.xml
builder: [CERT-TR] / [HIGH]
source:cert-tr / github, cve: / cwe: categories"] + ZEN["Zen Browser
RSS Dynamic Folder"] + + GH_A6 --> AGG + CT_B8 --> AGG + AGG --> CACHE + CACHE --> LOOP + CACHE --> REQ + LOOP -. poll & upsert .-> CACHE + REQ --> FEED --> ZEN + + style GH fill:#24292f,stroke:#1f2328,color:#fff + style CERT fill:#1e3a5f,stroke:#0f172a,color:#fff + style AGG fill:#f59e0b,stroke:#92400e,color:#000 + style CACHE fill:#eab308,stroke:#713f12,color:#000 + style LOOP fill:#6366f1,stroke:#312e81,color:#fff + style REQ fill:#06b6d4,stroke:#164e63,color:#fff + style FEED fill:#0ea5e9,stroke:#0c4a6e,color:#fff + style ZEN fill:#059669,stroke:#064e3b,color:#fff ``` -Key invariants: RSS reads cache only; GH enumeration is exhaustive; failures keep stale cache. +Key invariants: RSS reads cache only; GH enumeration is exhaustive; IMAP never sets `\Seen`; failures keep stale cache; `CERT-TR` strict CVE-only (hash fallback removed). --- -## Required GitHub Scopes/Permissions (Recap) +## Required Scopes/Permissions (Recap) -- **Fine-grained:** `Repository security advisories: Read`, `Metadata: Read`, (org bulk: `Organization security advisories: Read`) -- **Classic:** `repo` (or `public_repo` for public advisories only, but private/draft need `repo` / `repository_advisories:read`) +- **GitHub Fine-grained:** `Repository security advisories: Read`, `Metadata: Read`, (org bulk: `Organization security advisories: Read`) +- **GitHub Classic:** `repo` (or `public_repo` for public advisories only, but private/draft need `repo` / `repository_advisories:read`) +- **Proton Bridge:** Bridge must be running, `PROTON_BRIDGE_PASSWORD` is Bridge-generated (Mailbox details), not account password, paid plan required +- **Gmail App Password:** Google Account - 2FA - App Passwords - `GMAIL_APP_PASSWORD` (16 chars, spaces preserved), IMAP enabled +- **Gmail OAuth:** Google Cloud - OAuth Client ID (Desktop) - `GMAIL_OAUTH_CLIENT_ID`/`SECRET` - `app auth login --provider gmail --method oauth` - `refresh_token` stored in `cache/gmail_oauth.json` (600) + `.env` (`GMAIL_OAUTH_REFRESH_TOKEN(S)`), scope `https://mail.google.com/`, uses `XOAUTH2` `src/advisory_rss/auth/gmail.py:1` + `src/advisory_rss/cert_tr/imap.py:65` --- -## Known GitHub API Limitations +## Known Limitations -- **No server-side "created by me" filter** - all filtering is client-side after listing advisories per repo/state. Costs API calls (~1 per repo × 4 states + org bulk) and is rate-limited; mitigated with ETag 304, pagination caps, and background polling rather than per-RSS-call. -- **Enumeration requires listing all owned repos** (`GET /user/repos`) - large accounts (thousands of repos) need many pages; `MAX_REPOS`/`MAX_PAGES_PER_REPO` cap prevents runaway, `app status` reports caps hit. -- **Org bulk requires owner/security-manager** - else 403/404 per org and we fall back to per-repo enumeration (expected). -- **Global `GET /advisories` cannot tell ownership** - it is the public advisory database; ignored for discovery, used only as optional enrichment if needed. +- **GitHub No server-side "created by me" filter** - all filtering is client-side after listing advisories per repo/state. Costs API calls (~1 per repo × 4 states + org bulk) and is rate-limited; mitigated with ETag 304, pagination caps, and background polling rather than per-RSS-call. +- **GitHub Enumeration requires listing all owned repos** (`GET /user/repos`) - large accounts (thousands of repos) need many pages; `MAX_REPOS`/`MAX_PAGES_PER_REPO` cap prevents runaway, `app status` reports caps hit. +- **GitHub Org bulk requires owner/security-manager** - else 403/404 per org and we fall back to per-repo enumeration (expected). +- **GitHub Global `GET /advisories` cannot tell ownership** - it is the public advisory database; ignored for discovery, used only as optional enrichment if needed. - **GHSA != chronological** - must sort by `updated_at`, not ID. - **GraphQL `SecurityAdvisory` has no `author`** - verified; cannot replace REST for this use case. - **Global enrichment 404 for draft/private** - expected; we treat missing as not-enriched. - **Withdrawn state** differs between global (`withdrawn_at`/`is_withdrawn`) and repo (`state: closed/withdrawn` + `withdrawn_at`) - normalized into single `withdrawn_at`/`state`. +- **CERT-TR IMAP `FROM` search is substring** - `FROM "siberguvenlik.gov.tr"` is server-side substring, but final `is_cert_tr_sender()` `src/advisory_rss/cert_tr/parser.py:429` does strict domain check; non-CVE mails are now skipped entirely. +- **Gmail/Proton folders with non-ASCII** (e.g., `CVE Yazışmaları`) require `modified UTF-7` fallback `src/advisory_rss/cert_tr/imap.py:24`; `GMAIL_IMAP_FOLDERS` must be comma-separated (spaces preserved), not whitespace-split. +- **Gmail OAuth `refresh_token` only on first consent** - if `run_gmail_oauth_flow` `src/advisory_rss/auth/gmail.py:210` returns no `refresh_token`, revoke access at and retry with `prompt=consent`. +- **Proton Bridge self-signed cert** - `CERT_NONE` + `STARTTLS` (or `SSL`); Gmail uses `CERT_REQUIRED`. +- **Legacy `CERT-TR-MSG-*` rows** - auto-deleted on `CacheStore` init `src/advisory_rss/cache/store.py:76` (non-CVE false positives). --- ## Development & Tests ```bash -pytest -q # all 60 tests +pytest -q # 78 tests (60 original + 14 CERT-TR + 4 Gmail) pytest tests/unit -q -v pytest tests/integration -v # starts ephemeral server on 127.0.0.1:0, validates RSS & headers +pytest tests/unit/test_cert_tr_settings.py tests/unit/test_cert_tr_imap.py -v # Gmail/Proton multi-folder, OAuth pip-audit # audit scan # Regenerate real fixtures from live GitHub API (uses GITHUB_TOKEN from .env, auto-detects GITHUB_REPOS/GITHUB_ORG/SKIP_FULL_SCAN) python scripts/fixtures_generator.py python scripts/fixtures_generator.py --ghsa GHSA-xxxx-xxxx-xxxx --repo bla/bla # single GHSA -# Lint / type-check (standard, line-based ignores only - no config-based ignore) +# Lint / type-check (no noqa, truly fixed) ruff check # All checks passed! pyright # 0 errors, typeCheckingMode: standard, extraPaths: ["src"] ``` ---- - ## License -GNU \ No newline at end of file +GNU \ No newline at end of file diff --git a/src/advisory_rss/auth/gmail.py b/src/advisory_rss/auth/gmail.py index 8009b5c..509f638 100644 --- a/src/advisory_rss/auth/gmail.py +++ b/src/advisory_rss/auth/gmail.py @@ -1,4 +1,4 @@ -"""Gmail OAuth helpers for CERT-TR IMAP (XOAUTH2).""" +#Gmail OAuth helpers from __future__ import annotations @@ -80,34 +80,46 @@ def get_refresh_token_for_email( email: str, settings: Any, ) -> tuple[str | None, str | None, str | None]: - """Return (refresh_token, client_id, client_secret) for email. - - Checks env vars first (GMAIL_OAUTH_REFRESH_TOKEN etc), then file. - For multi-account, checks GMAIL_OAUTH_REFRESH_TOKENS aligned with GMAIL_EMAILS. - """ + #Return (refresh_token, client_id, client_secret) for email. + #Checks env vars first (GMAIL_OAUTH_REFRESH_TOKEN etc), then file. + #For multi-account, checks GMAIL_OAUTH_REFRESH_TOKENS aligned with GMAIL_EMAILS. # Check env single/multi # settings may have gmail_oauth_refresh_token(s) and client_id/secret try: - # Use get_gmail_accounts logic? But we can directly check settings - # Check file first for per-account - file_data = _load_oauth_file(getattr(settings, "gmail_oauth_token_file", DEFAULT_GMAIL_OAUTH_CACHE)) + file_data = _load_oauth_file( + getattr(settings, "gmail_oauth_token_file", DEFAULT_GMAIL_OAUTH_CACHE) + ) # Try file per-account accounts = file_data.get("accounts") if isinstance(file_data, dict) else None if isinstance(accounts, dict) and email in accounts: info = accounts[email] if isinstance(info, dict) and info.get("refresh_token"): cid = file_data.get("client_id") or getattr(settings, "gmail_oauth_client_id", None) - csec = file_data.get("client_secret") or getattr(settings, "gmail_oauth_client_secret", None) - return str(info["refresh_token"]), str(cid or "") if cid else None, str(csec or "") if csec else None + csec = file_data.get("client_secret") or getattr( + settings, "gmail_oauth_client_secret", None + ) + return ( + str(info["refresh_token"]), + str(cid or "") if cid else None, + str(csec or "") if csec else None, + ) if isinstance(info, str): cid = file_data.get("client_id") or getattr(settings, "gmail_oauth_client_id", None) - csec = file_data.get("client_secret") or getattr(settings, "gmail_oauth_client_secret", None) + csec = file_data.get("client_secret") or getattr( + settings, "gmail_oauth_client_secret", None + ) return info, str(cid or "") if cid else None, str(csec or "") if csec else None # Check flat file if isinstance(file_data, dict) and email in file_data and isinstance(file_data[email], str): cid = file_data.get("client_id") or getattr(settings, "gmail_oauth_client_id", None) - csec = file_data.get("client_secret") or getattr(settings, "gmail_oauth_client_secret", None) - return str(file_data[email]), str(cid or "") if cid else None, str(csec or "") if csec else None + csec = file_data.get("client_secret") or getattr( + settings, "gmail_oauth_client_secret", None + ) + return ( + str(file_data[email]), + str(cid or "") if cid else None, + str(csec or "") if csec else None, + ) # Check env multi gmail_emails = getattr(settings, "gmail_emails", None) @@ -117,35 +129,40 @@ def get_refresh_token_for_email( import re emails = [e.strip() for e in re.split(r"[,\s;]+", gmail_emails) if e.strip()] - # For refresh tokens, split on comma only (preserve spaces? refresh tokens have no spaces) rts = [r.strip() for r in gmail_refresh_tokens.split(",") if r.strip()] try: idx = [e.lower() for e in emails].index(email.lower()) if idx < len(rts): cid = getattr(settings, "gmail_oauth_client_id", None) csec = getattr(settings, "gmail_oauth_client_secret", None) - return rts[idx], str(cid or "") if cid else None, str(csec or "") if csec else None + return ( + rts[idx], + str(cid or "") if cid else None, + str(csec or "") if csec else None, + ) except ValueError: pass # Check env single single_rt = getattr(settings, "gmail_oauth_refresh_token", None) if single_rt and single_rt.strip(): - # If single email matches or only one email configured - # If settings has single gmail_email matching, use it; else if only one refresh token and email matches gmail_email gmail_email = getattr(settings, "gmail_email", None) if gmail_email and gmail_email.strip().lower() == email.lower(): cid = getattr(settings, "gmail_oauth_client_id", None) csec = getattr(settings, "gmail_oauth_client_secret", None) - return single_rt.strip(), str(cid or "") if cid else None, str(csec or "") if csec else None - # If no specific email, but we have refresh token and email is the one being queried, return it - # This handles case where get_gmail_accounts already built with refresh token + return ( + single_rt.strip(), + str(cid or "") if cid else None, + str(csec or "") if csec else None, + ) cid = getattr(settings, "gmail_oauth_client_id", None) csec = getattr(settings, "gmail_oauth_client_secret", None) - # Only return if email is in gmail accounts list or if no list - # For safety, return if we have a refresh token and no other mapping if not gmail_emails: - return single_rt.strip(), str(cid or "") if cid else None, str(csec or "") if csec else None + return ( + single_rt.strip(), + str(cid or "") if cid else None, + str(csec or "") if csec else None, + ) except (OSError, ValueError, AttributeError) as e: logger.debug("get_refresh_token failed for %s: %s", email, e) return None, None, None @@ -157,11 +174,7 @@ def fetch_access_token( client_secret: str, token_uri: str = GMAIL_OAUTH_TOKEN_URI, ) -> tuple[str, int | None]: - """Exchange refresh token for access token via Google OAuth2. - - Returns (access_token, expires_in_seconds). - Raises on failure. - """ + #Exchange refresh token for access token via Google OAuth2. if not refresh_token or not client_id or not client_secret: raise ValueError("Missing OAuth credentials (refresh_token/client_id/secret)") @@ -200,12 +213,12 @@ def run_gmail_oauth_flow( client_secret: str, email: str | None = None, ) -> tuple[str, str, str | None]: - """Run InstalledAppFlow to obtain refresh token. + #Run InstalledAppFlow to obtain refresh token. - Returns (refresh_token, access_token, expiry). - Requires google-auth-oauthlib to be installed. - Opens browser for user consent. - """ + #Returns (refresh_token, access_token, expiry). + #Requires google-auth-oauthlib to be installed. + #Opens browser for user consent. + try: from google_auth_oauthlib.flow import InstalledAppFlow except ImportError as e: @@ -225,10 +238,6 @@ def run_gmail_oauth_flow( } flow = InstalledAppFlow.from_client_config(client_config, scopes=GMAIL_OAUTH_SCOPES) - # Optional: set login hint if email provided - # The flow will open browser; we use local server - # Use port 0 to get random available port, or 8080 - # prompt=consent to ensure refresh_token is returned creds = flow.run_local_server( port=0, prompt="consent", @@ -240,9 +249,10 @@ def run_gmail_oauth_flow( expiry = getattr(creds, "expiry", None) expiry_str = expiry.isoformat() if expiry else None if not refresh_token: - # Sometimes refresh_token is only returned on first consent; if not, we try to use existing - # But we should warn - logger.warning("OAuth flow did not return a refresh_token (maybe already consented). Try revoking access and retrying.") - # Fallback: try to use current refresh token if available? For now, raise - raise ValueError("No refresh_token returned from OAuth flow. Ensure you revoke previous consent in Google Account and retry with prompt=consent.") + logger.warning( + "OAuth flow did not return a refresh_token (maybe already consented). Try revoking access and retrying." + ) + raise ValueError( + "No refresh_token returned from OAuth flow. Ensure you revoke previous consent in Google Account and retry with prompt=consent." + ) return str(refresh_token), str(access_token or ""), expiry_str diff --git a/src/advisory_rss/cache/models.py b/src/advisory_rss/cache/models.py index d8f2a86..42856f2 100644 --- a/src/advisory_rss/cache/models.py +++ b/src/advisory_rss/cache/models.py @@ -1,5 +1,3 @@ -"""Normalized advisory dataclass and cache meta.""" - from __future__ import annotations from dataclasses import asdict, dataclass, field diff --git a/src/advisory_rss/cache/store.py b/src/advisory_rss/cache/store.py index 3bf6520..305f7c9 100644 --- a/src/advisory_rss/cache/store.py +++ b/src/advisory_rss/cache/store.py @@ -1,4 +1,4 @@ -"""Persistent cache SQLite with WAL, parameterized queries, ETag map, stale fallback.""" +# Persistent cache SQLite with WAL, parameterized queries, ETag map, stale fallback from __future__ import annotations @@ -15,9 +15,6 @@ _lock = threading.Lock() -# Never log token - this store never touches token - - def _connect(db_path: Path) -> sqlite3.Connection: db_path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect( @@ -69,11 +66,15 @@ def _init_schema(self) -> None: self._conn.executescript(SCHEMA) # Lightweight migrations for existing DBs (idempotent) try: - cols = {r[1] for r in self._conn.execute("PRAGMA table_info(advisories)").fetchall()} + cols = { + r[1] for r in self._conn.execute("PRAGMA table_info(advisories)").fetchall() + } if "source" not in cols: - self._conn.execute("ALTER TABLE advisories ADD COLUMN source TEXT DEFAULT 'github'") + self._conn.execute( + "ALTER TABLE advisories ADD COLUMN source TEXT DEFAULT 'github'" + ) logger.info("Migrated advisories: added source column") - # Backfill normalized_json missing source field (always, idempotent) + # Backfill normalized_json missing source field try: cur = self._conn.execute("SELECT ghsa_id, normalized_json FROM advisories") for ghsa_id, nj in cur.fetchall(): @@ -92,7 +93,9 @@ def _init_schema(self) -> None: logger.debug("backfill query failed: %s", ee) # Cleanup legacy false positives: non-CVE CERT-TR entries (hash-based IDs) try: - cur = self._conn.execute("DELETE FROM advisories WHERE ghsa_id LIKE 'CERT-TR-MSG-%'") + cur = self._conn.execute( + "DELETE FROM advisories WHERE ghsa_id LIKE 'CERT-TR-MSG-%'" + ) if cur.rowcount and cur.rowcount > 0: logger.info("Cleaned up %d legacy CERT-TR non-CVE advisories", cur.rowcount) except sqlite3.Error as ce: @@ -293,8 +296,6 @@ def mark_success(self, user: str | None = None) -> None: self.set_meta("rate_limited_until", None) if user: self.set_meta("authenticated_user", user) - # compute next - # caller sets next_scheduled via interval; but also set here roughly self.set_meta("advisories_count", str(self.count())) logger.info("cache mark_success user=%s time=%s", user or "unknown", now) diff --git a/src/advisory_rss/cert_tr/__init__.py b/src/advisory_rss/cert_tr/__init__.py index 858031d..4bf05a3 100644 --- a/src/advisory_rss/cert_tr/__init__.py +++ b/src/advisory_rss/cert_tr/__init__.py @@ -1 +1 @@ -"""CERT-TR / SGB Proton Mail source package.""" +#CERT-TR / SGB Proton Mail source package \ No newline at end of file diff --git a/src/advisory_rss/cert_tr/imap.py b/src/advisory_rss/cert_tr/imap.py index 6c6e628..b9f87cb 100644 --- a/src/advisory_rss/cert_tr/imap.py +++ b/src/advisory_rss/cert_tr/imap.py @@ -1,4 +1,4 @@ -"""IMAP fetcher for CERT-TR mails via Proton Bridge and Gmail (never marks read).""" +#IMAP fetcher for CERT-TR mails via Proton Bridge and Gmail from __future__ import annotations @@ -21,9 +21,6 @@ "imap.googlemail.com", } -# Never mark read: we always use BODY.PEEK and SELECT readonly=True - - def _ssl_context_for_host(host: str) -> ssl.SSLContext: # Proton Bridge uses self-signed cert -> no verification if host in {"127.0.0.1", "::1", "localhost"}: @@ -64,7 +61,7 @@ def connect_imap(account: dict[str, str]) -> imaplib.IMAP4: else: # NONE mail = imaplib.IMAP4(host, port) - # Gmail OAuth (XOAUTH2) if configured + # Gmail OAuth if configured if auth_method == "oauth" and oauth_refresh_token: # Need client_id/secret if not oauth_client_id or not oauth_client_secret: @@ -77,12 +74,18 @@ def connect_imap(account: dict[str, str]) -> imaplib.IMAP4: ) xoauth2 = build_xoauth2_string(email, access_token) - # imaplib authenticate with XOAUTH2 def _auth_cb(_: bytes) -> str: return xoauth2 mail.authenticate("XOAUTH2", _auth_cb) # type: ignore[arg-type] - except (OSError, ValueError, RuntimeError, imaplib.IMAP4.error, httpx.HTTPError, ImportError) as e: + except ( + OSError, + ValueError, + RuntimeError, + imaplib.IMAP4.error, + httpx.HTTPError, + ImportError, + ) as e: logger.error("IMAP XOAUTH2 failed for %s: %s", email, e) raise else: @@ -97,7 +100,6 @@ def _auth_cb(_: bytes) -> str: def _encode_modified_utf7(s: str) -> str: - # IMAP modified UTF-7 (RFC 3501): encode non-ASCII runs as &- where base64 uses , for / import base64 res: list[str] = [] @@ -133,16 +135,12 @@ def _b64(buf_chars: list[str]) -> str: return "".join(res) -def _select_folder_imap(mail: imaplib.IMAP4, folder: str, readonly: bool = True) -> tuple[str, list[bytes]]: - # Try UTF-8 first (if server supports UTF8=ACCEPT), then modified UTF-7 - # Gmail supports both, but Proton Bridge is ASCII-only, so INBOX will succeed directly. - # For non-ASCII like "CVE Yazışmaları", try UTF-8, then fallback to modified UTF-7. - # Try with mail._encoding = 'utf-8' +def _select_folder_imap( + mail: imaplib.IMAP4, folder: str, readonly: bool = True +) -> tuple[str, list[bytes]]: orig_enc = getattr(mail, "_encoding", "ascii") # First try: enable UTF8 if possible try: - # Some servers require ENABLE UTF8=ACCEPT before UTF-8 mailbox names - # Ignore failure try: mail.enable("UTF8=ACCEPT") except (imaplib.IMAP4.error, OSError, ValueError): @@ -197,11 +195,6 @@ def fetch_raw_emails( max_mails: int = 200, search_days: int | None = None, ) -> list[bytes]: - """Fetch raw RFC822 bytes for one account, never marking as read. - - Returns list of raw bytes, newest first (reverse UID). - Handles reconnect every 25 to avoid Bridge drops. - """ folder = account.get("folder", "INBOX") or "INBOX" email = account.get("email", "unknown") raw_list: list[bytes] = [] @@ -270,7 +263,13 @@ def fetch_raw_emails( try: mail = connect_imap(account) _select_folder_imap(mail, folder, readonly=True) - except (OSError, imaplib.IMAP4.error, ssl.SSLError, ValueError, UnicodeEncodeError) as e: + except ( + OSError, + imaplib.IMAP4.error, + ssl.SSLError, + ValueError, + UnicodeEncodeError, + ) as e: logger.warning("IMAP reconnect failed %s batch %d: %s", email, i, e) break @@ -281,7 +280,11 @@ def fetch_raw_emails( if typ != "OK" or not fdata: continue for item in fdata: - if isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], bytes): + if ( + isinstance(item, tuple) + and len(item) == 2 + and isinstance(item[1], bytes) + ): raw_list.append(item[1]) fetched += 1 elif isinstance(item, bytes) and len(item) > 100: diff --git a/src/advisory_rss/cert_tr/parser.py b/src/advisory_rss/cert_tr/parser.py index 566ba1d..b86e4df 100644 --- a/src/advisory_rss/cert_tr/parser.py +++ b/src/advisory_rss/cert_tr/parser.py @@ -34,7 +34,6 @@ # Sender allowlist check is done outside but parser also extracts domain - def decode_rfc2047(s: str | None) -> str: if not s: return "" @@ -54,7 +53,6 @@ def decode_rfc2047(s: str | None) -> str: logger.debug("decode_rfc2047 failed %r: %s", s[:80], e) return str(s) - def _extract_domain(from_header: str) -> str: # from_header already decoded may contain m = re.search(r"<([^>]+)>", from_header or "") @@ -64,7 +62,6 @@ def _extract_domain(from_header: str) -> str: return addr.split("@", 1)[1].lower() return addr - def _parse_date(date_str: str | None) -> datetime | None: if not date_str: return None @@ -77,7 +74,6 @@ def _parse_date(date_str: str | None) -> datetime | None: logger.debug("date parse failed %r: %s", date_str, e) return None - def _html_to_text(html_body: str) -> str: # Lightweight html -> text: strip tags, decode entities, keep line breaks # Replace
,

,

with newlines before stripping @@ -101,9 +97,7 @@ def _html_to_text(html_body: str) -> str: s = re.sub(r" *\n *", "\n", s) return s.strip() - def _extract_body(msg: EmailMessage) -> tuple[str, str | None]: - """Return (plain_text, html_raw_or_None). Decodes transfer encodings.""" # Walk parts; prefer html then plain; handle multipart plain: str | None = None html_raw: str | None = None @@ -170,7 +164,6 @@ def _extract_body(msg: EmailMessage) -> tuple[str, str | None]: return _html_to_text(text), text return text, None - def _extract_cves(text: str) -> list[str]: found = CVE_RE.findall(text or "") # normalize upper @@ -184,12 +177,10 @@ def _extract_cves(text: str) -> list[str]: out.append(c) return out - def _extract_cwe(text: str) -> str | None: m = CWE_RE.search(text or "") return m.group(0).upper() if m else None - def _extract_product(text: str) -> str | None: if not text: return None @@ -218,8 +209,9 @@ def _extract_product(text: str) -> str | None: return candidates[0].strip() return None - -def _build_summary(subject_decoded: str, product: str | None, cve: str | None, cwe: str | None) -> str: +def _build_summary( + subject_decoded: str, product: str | None, cve: str | None, cwe: str | None +) -> str: # If product + CVE, build canonical summary, else decoded subject if product and cve: base = f"{product}: {cve}" @@ -240,7 +232,6 @@ def _build_summary(subject_decoded: str, product: str | None, cve: str | None, c return s return subject_decoded.strip() or "CERT-TR Zafiyet Bildirimi" - def parse_cert_tr_email(raw_bytes: bytes) -> NormalizedAdvisory | None: """Parse raw RFC822 bytes from Proton/Bridge or Gmail into NormalizedAdvisory. @@ -279,8 +270,6 @@ def parse_cert_tr_email(raw_bytes: bytes) -> NormalizedAdvisory | None: cwe = _extract_cwe(combined_text) product = _extract_product(combined_text) - # If no CVE found, still maybe useful? Require at least product or cwe or cve - # For strict CERT-TR, require CVE; otherwise skip to avoid noise from unrelated mail if not cve_primary and message_id: m = CVE_RE.search(message_id) if m: @@ -296,7 +285,6 @@ def parse_cert_tr_email(raw_bytes: bytes) -> NormalizedAdvisory | None: ) return None - # ID generation: strictly CVE-based (prevents false positives) ghsa_id = f"CERT-TR-{cve_primary.upper()}" # Extra CVEs beyond primary @@ -310,8 +298,6 @@ def parse_cert_tr_email(raw_bytes: bytes) -> NormalizedAdvisory | None: description = _html_to_text(html_raw) if not description: description = combined_text[:2000] - # Include product/vendor context if product found and not in summary - # Keep within model limits; no hard truncation here, builder does it # References refs: list[str] = [] @@ -321,7 +307,7 @@ def parse_cert_tr_email(raw_bytes: bytes) -> NormalizedAdvisory | None: # also include siberguvenlik portal (generic) refs.append("https://siberguvenlik.gov.tr") - # Links: CVE-based NVD link (strict CVE-only) + # Links: CVE-based NVD link html_url = f"https://nvd.nist.gov/vuln/detail/{cve_primary}" # Severity: try to infer from text (kritik/yüksek/orta/düşük) diff --git a/src/advisory_rss/cert_tr/source.py b/src/advisory_rss/cert_tr/source.py index d09f7e9..d0faee4 100644 --- a/src/advisory_rss/cert_tr/source.py +++ b/src/advisory_rss/cert_tr/source.py @@ -1,4 +1,4 @@ -"""CERT-TR source orchestrator: IMAP fetch -> parser -> NormalizedAdvisory list.""" +#CERT-TR source orchestrator: IMAP fetch -> parser -> NormalizedAdvisory list. from __future__ import annotations @@ -19,11 +19,7 @@ def __init__(self, settings: Settings): self.settings = settings def fetch_all(self) -> tuple[list[NormalizedAdvisory], dict[str, Any]]: - """Fetch from all configured Proton accounts. - - Returns (advisories, diag). Never raises (errors go to diag). - Dedup by ghsa_id (CERT-TR-CVE-...) keep newest (by updated_at). - """ + #Fetch from all configured Proton accounts. diag: dict[str, Any] = { "accounts": 0, "raw_fetched": 0, @@ -56,7 +52,9 @@ def fetch_all(self) -> tuple[list[NormalizedAdvisory], dict[str, Any]]: folder = acc.get("folder", "INBOX") provider = acc.get("provider", "unknown") if not acc.get("password"): - msg = f"{provider} account {email}/{folder}: missing password/app-password - skipping" + msg = ( + f"{provider} account {email}/{folder}: missing password/app-password - skipping" + ) logger.warning(msg) diag["errors"].append(msg) continue diff --git a/src/advisory_rss/cli.py b/src/advisory_rss/cli.py index eb71b3d..26d9bd8 100644 --- a/src/advisory_rss/cli.py +++ b/src/advisory_rss/cli.py @@ -37,7 +37,6 @@ def configure_logging( force: bool = False, use_stderr: bool = True, ) -> None: - """Backwards-compat wrapper around centralized logging_config.setup_logging.""" try: settings = get_settings() # Prefer explicit args, else settings @@ -62,7 +61,6 @@ def auth() -> None: def _upsert_env_file(key: str, value: str, env_path: str = ".env") -> None: - """Create or update .env file with key=value (preserves other lines).""" import re as _re2 p = Path(env_path) @@ -73,7 +71,7 @@ def _upsert_env_file(key: str, value: str, env_path: str = ".env") -> None: except OSError as e: logger.debug("Failed to read %s: %s", env_path, e) lines = [] - # Find existing key (case-insensitive for env var name, but keep case) + # Find existing key found = False new_lines: list[str] = [] pattern = _re2.compile(rf"^\s*{_re2.escape(key)}\s*=", _re.IGNORECASE) @@ -130,7 +128,9 @@ def _prompt_gmail_method(default: str | None = None) -> str: help="Gmail için yöntem (oauth veya app-password). Sadece gmail için.", ) @click.option("--email", default=None, help="E-posta adresi (proton/gmail için)") -def auth_login(provider: str | None = None, method: str | None = None, email: str | None = None) -> None: +def auth_login( + provider: str | None = None, method: str | None = None, email: str | None = None +) -> None: settings = get_settings() _setup_logging(settings.log_level, log_file=settings.log_file, log_format=settings.log_format) logger.info("auth login started provider=%s method=%s email=%s", provider, method, email) @@ -141,8 +141,12 @@ def auth_login(provider: str | None = None, method: str | None = None, email: st provider = provider.lower() if provider == "github": - click.echo("GitHub PAT: fine-grained 'Repository security advisories: Read' is recommended.") - click.echo("Token must be set in .env as GITHUB_TOKEN - credentials file is no longer used.") + click.echo( + "GitHub PAT: fine-grained 'Repository security advisories: Read' is recommended." + ) + click.echo( + "Token must be set in .env as GITHUB_TOKEN - credentials file is no longer used." + ) existing_env = settings.token if existing_env: click.echo(f"Note: GITHUB_TOKEN already set via env ({token_preview(existing_env)}).") @@ -196,14 +200,18 @@ async def _validate() -> dict: if provider == "proton": # Proton Bridge: email + bridge password if not email: - email = click.prompt("Proton e-posta adresi", type=str, default=settings.proton_bridge_email or "") + email = click.prompt( + "Proton e-posta adresi", type=str, default=settings.proton_bridge_email or "" + ) email = email.strip() if not email or "@" not in email: click.echo("Geçersiz e-posta", err=True) sys.exit(1) # Ask for bridge password (hidden) try: - bridge_pw = getpass.getpass(f"Proton Bridge şifresi ({email}) (Bridge > Mailbox details, NOT account password): ").strip() + bridge_pw = getpass.getpass( + f"Proton Bridge şifresi ({email}) (Bridge > Mailbox details, NOT account password): " + ).strip() except (EOFError, KeyboardInterrupt): click.echo("\nCancelled.", err=True) sys.exit(1) @@ -232,10 +240,14 @@ async def _validate() -> dict: mail.logout() except (OSError, ValueError, RuntimeError) as e: logger.debug("Proton logout failed: %s", e) - click.echo(f"+ Proton IMAP doğrulandı: {email} -> {acc['folder']} @ {acc['host']}:{acc['port']}") + click.echo( + f"+ Proton IMAP doğrulandı: {email} -> {acc['folder']} @ {acc['host']}:{acc['port']}" + ) except (OSError, ValueError, RuntimeError) as e: click.echo(f"- Proton IMAP hatası: {_redact(str(e))}", err=True) - click.echo("Bridge çalışıyor mu? Host/port doğru mu? Şifre Bridge şifresi mi?", err=True) + click.echo( + "Bridge çalışıyor mu? Host/port doğru mu? Şifre Bridge şifresi mi?", err=True + ) sys.exit(1) # Save to .env - handle multi @@ -257,7 +269,9 @@ async def _validate() -> dict: else: new_pw = bridge_pw _upsert_env_file("PROTON_BRIDGE_PASSWORDS", new_pw) - click.echo(f"Wrote PROTON_BRIDGE_EMAILS and PROTON_BRIDGE_PASSWORDS to .env (appended {email})") + click.echo( + f"Wrote PROTON_BRIDGE_EMAILS and PROTON_BRIDGE_PASSWORDS to .env (appended {email})" + ) else: _upsert_env_file("PROTON_BRIDGE_EMAIL", email) _upsert_env_file("PROTON_BRIDGE_PASSWORD", bridge_pw) @@ -277,7 +291,9 @@ async def _validate() -> dict: gmail_method = "app-password" if not email: - default_email = settings.gmail_email or (settings.gmail_emails.split(",")[0].strip() if settings.gmail_emails else "") + default_email = settings.gmail_email or ( + settings.gmail_emails.split(",")[0].strip() if settings.gmail_emails else "" + ) email = click.prompt("Gmail e-posta adresi", type=str, default=default_email) email = email.strip() if not email or "@" not in email: @@ -286,7 +302,9 @@ async def _validate() -> dict: if gmail_method == "app-password": try: - app_pw = getpass.getpass(f"Gmail App Password ({email}) (16 haneli, Google Hesabı > Güvenlik > 2FA > Uygulama şifreleri): ").strip() + app_pw = getpass.getpass( + f"Gmail App Password ({email}) (16 haneli, Google Hesabı > Güvenlik > 2FA > Uygulama şifreleri): " + ).strip() except (EOFError, KeyboardInterrupt): click.echo("\nCancelled.", err=True) sys.exit(1) @@ -316,7 +334,10 @@ async def _validate() -> dict: click.echo(f"+ Gmail IMAP (App Password) doğrulandı: {email}") except (OSError, ValueError, RuntimeError) as e: click.echo(f"- Gmail hatası: {_redact(str(e))}", err=True) - click.echo("App Password doğru mu? 2FA açık mı? IMAP enabled mi? (Gmail > Ayarlar > Yönlendirme ve POP/IMAP)", err=True) + click.echo( + "App Password doğru mu? 2FA açık mı? IMAP enabled mi? (Gmail > Ayarlar > Yönlendirme ve POP/IMAP)", + err=True, + ) sys.exit(1) # Save to .env (handle multi) @@ -335,7 +356,9 @@ async def _validate() -> dict: else: new_pw = app_pw _upsert_env_file("GMAIL_APP_PASSWORDS", new_pw) - click.echo(f"Wrote GMAIL_EMAILS and GMAIL_APP_PASSWORDS to .env (appended {email}, spaces preserved)") + click.echo( + f"Wrote GMAIL_EMAILS and GMAIL_APP_PASSWORDS to .env (appended {email}, spaces preserved)" + ) else: _upsert_env_file("GMAIL_EMAIL", email) _upsert_env_file("GMAIL_APP_PASSWORD", app_pw) @@ -349,11 +372,15 @@ async def _validate() -> dict: client_secret = settings.gmail_oauth_client_secret if not client_id or not client_secret: click.echo("Gmail OAuth için Google Cloud OAuth Client ID/Secret gerekli.") - click.echo("Oluştur: https://console.cloud.google.com/ -> APIs & Services -> Credentials -> Create Credentials -> OAuth Client ID (Desktop)") + click.echo( + "Oluştur: https://console.cloud.google.com/ -> APIs & Services -> Credentials -> Create Credentials -> OAuth Client ID (Desktop)" + ) click.echo("Redirect URI: http://localhost (InstalledAppFlow otomatik)") click.echo("Scopes: https://mail.google.com/") if not client_id: - client_id = click.prompt("OAuth Client ID", type=str, default=client_id or "").strip() + client_id = click.prompt( + "OAuth Client ID", type=str, default=client_id or "" + ).strip() if not client_secret: try: client_secret = getpass.getpass("OAuth Client Secret (hidden): ").strip() @@ -379,7 +406,9 @@ async def _validate() -> dict: sys.exit(1) try: - refresh_token, access_token, expiry = run_gmail_oauth_flow(client_id, client_secret, email) + refresh_token, access_token, expiry = run_gmail_oauth_flow( + client_id, client_secret, email + ) click.echo(f"+ OAuth başarılı: refresh_token alındı ({refresh_token[:8]}...)") except (OSError, ValueError, RuntimeError) as e: click.echo(f"- OAuth hatası: {_redact(str(e))}", err=True) @@ -418,7 +447,9 @@ async def _validate() -> dict: # Prefer file cache plus env token_file = settings.gmail_oauth_token_file or "cache/gmail_oauth.json" try: - save_gmail_oauth_token(token_file, email, refresh_token, client_id, client_secret, access_token, expiry) + save_gmail_oauth_token( + token_file, email, refresh_token, client_id, client_secret, access_token, expiry + ) click.echo(f"Wrote OAuth token to {token_file} (600 perms)") except (OSError, ValueError, RuntimeError) as e: logger.debug("Failed to save to file: %s", e) @@ -438,7 +469,9 @@ async def _validate() -> dict: else: new_rts = refresh_token _upsert_env_file("GMAIL_OAUTH_REFRESH_TOKENS", new_rts) - click.echo(f"Wrote GMAIL_EMAILS and GMAIL_OAUTH_REFRESH_TOKENS to .env (appended {email})") + click.echo( + f"Wrote GMAIL_EMAILS and GMAIL_OAUTH_REFRESH_TOKENS to .env (appended {email})" + ) elif existing_emails: # Update existing entry # Find index and replace @@ -469,8 +502,12 @@ async def _validate() -> dict: _upsert_env_file("GMAIL_OAUTH_REFRESH_TOKEN", refresh_token) click.echo("Wrote GMAIL_EMAIL and GMAIL_OAUTH_REFRESH_TOKEN to .env") - click.echo("OAuth tamamlandı. ENABLE_CERT_TR=true olduğundan emin olun, sonra: app sync --source cert-tr") - click.echo("Not: Access token otomatik yenilenecek (refresh_token saklandı). İptal için Google Hesabı > Güvenlik > Üçüncü taraf erişimi'nden kaldırın.") + click.echo( + "OAuth tamamlandı. ENABLE_CERT_TR=true olduğundan emin olun, sonra: app sync --source cert-tr" + ) + click.echo( + "Not: Access token otomatik yenilenecek (refresh_token saklandı). İptal için Google Hesabı > Güvenlik > Üçüncü taraf erişimi'nden kaldırın." + ) return click.echo(f"Bilinmeyen provider: {provider}", err=True) @@ -509,7 +546,10 @@ def sync(source: str = "all") -> None: if need_github and not token: # If CERT-TR is enabled and user asked all, still allow cert-tr part to run if settings.enable_cert_tr and source == "all": - click.echo("Warning: GITHUB_TOKEN not set - GitHub sync will be skipped, CERT-TR only.", err=True) + click.echo( + "Warning: GITHUB_TOKEN not set - GitHub sync will be skipped, CERT-TR only.", + err=True, + ) logger.warning("sync without GitHub token - GitHub skipped, CERT-TR only") need_github = False else: @@ -592,10 +632,14 @@ async def _run() -> int: combined_diag["errors"].extend(diag_ct.get("errors") or []) if advs_ct: all_advisories.extend(advs_ct) - click.echo(f"CERT-TR sync OK - {len(advs_ct)} advisories from {diag_ct.get('accounts')} account(s) (raw {diag_ct.get('raw_fetched')})") + click.echo( + f"CERT-TR sync OK - {len(advs_ct)} advisories from {diag_ct.get('accounts')} account(s) (raw {diag_ct.get('raw_fetched')})" + ) click.echo(f"Latest CERT-TR: {advs_ct[0].ghsa_id} - {advs_ct[0].summary[:80]}") else: - click.echo(f"CERT-TR sync: no advisories (raw {diag_ct.get('raw_fetched')}, filtered_sender {diag_ct.get('filtered_sender')})") + click.echo( + f"CERT-TR sync: no advisories (raw {diag_ct.get('raw_fetched')}, filtered_sender {diag_ct.get('filtered_sender')})" + ) if diag_ct.get("errors"): for e in diag_ct["errors"][:3]: click.echo(f" CERT-TR warn: {_redact(e)}", err=True) @@ -613,7 +657,9 @@ async def _run() -> int: # --- Upsert combined --- if not all_advisories: # Check if both sources were skipped vs empty result - click.echo("No advisories from any source - cache unchanged. Check GITHUB_TOKEN / PROTON_* settings.") + click.echo( + "No advisories from any source - cache unchanged. Check GITHUB_TOKEN / PROTON_* settings." + ) # Still mark success if no error (to update next sync) if not combined_diag["errors"]: cache.mark_success(user=cache.get_meta("authenticated_user")) @@ -640,10 +686,14 @@ async def _run() -> int: import json if combined_diag.get("cert_tr"): - cache.set_meta("cert_tr_diag", json.dumps(combined_diag["cert_tr"], ensure_ascii=False)) + cache.set_meta( + "cert_tr_diag", json.dumps(combined_diag["cert_tr"], ensure_ascii=False) + ) except (OSError, ValueError, TypeError, RuntimeError) as e: logger.debug("Failed to persist cert_tr_diag: %s", e) - click.echo(f"Sync OK - {count} advisories upserted total ({len(all_advisories)} fetched before dedup)") + click.echo( + f"Sync OK - {count} advisories upserted total ({len(all_advisories)} fetched before dedup)" + ) if combined_diag["errors"]: click.echo(f"Warnings ({len(combined_diag['errors'])}):", err=True) for e in combined_diag["errors"][:5]: @@ -838,7 +888,7 @@ def serve( except OSError as e: logger.debug("stale pid cleanup failed: %s", _redact(str(e))) click.echo( - f"Daemonizing → pid: {pid_file}, log: {log_file} (terminal kapanınca da yaşar, nohup/setsid)" + f"Daemonizing - pid: {pid_file}, log: {log_file} (terminal kapanınca da yaşar, nohup/setsid)" ) _daemonize(pid_file, log_file) # Child continues; stdout/stderr now goes to log file @@ -977,7 +1027,9 @@ def _status() -> None: # Per-source breakdown try: all_adv = cache.load_all() - github_c = sum(1 for a in all_adv if (getattr(a, "source", "github") or "github") == "github") + github_c = sum( + 1 for a in all_adv if (getattr(a, "source", "github") or "github") == "github" + ) cert_c = sum(1 for a in all_adv if getattr(a, "source", "") == "cert-tr") click.echo(f" - GitHub: {github_c}") click.echo(f" - CERT-TR: {cert_c}") @@ -999,9 +1051,13 @@ def _status() -> None: for a in accs: # Never show password prov = a.get("provider", "unknown") - click.echo(f" - [{prov}] {a['email']} -> folder={a['folder']} host={a['host']}:{a['port']} sec={a['security']}") + click.echo( + f" - [{prov}] {a['email']} -> folder={a['folder']} host={a['host']}:{a['port']} sec={a['security']}" + ) click.echo(f"CERT-TR allowlist: {settings.cert_tr_sender_allowlist}") - click.echo(f"CERT-TR max mails: {settings.cert_tr_max_mails} search_days: {settings.cert_tr_search_days or 'all'}") + click.echo( + f"CERT-TR max mails: {settings.cert_tr_max_mails} search_days: {settings.cert_tr_search_days or 'all'}" + ) diag_raw = cache.get_meta("cert_tr_diag") if diag_raw: click.echo(f"CERT-TR last diag: {diag_raw[:300]}") diff --git a/src/advisory_rss/config/settings.py b/src/advisory_rss/config/settings.py index 0aa90d2..8d1097b 100644 --- a/src/advisory_rss/config/settings.py +++ b/src/advisory_rss/config/settings.py @@ -109,9 +109,7 @@ def __init__( ) # Cert-TR / Proton Mail (multi-account, per-mailbox folder) - enable_cert_tr: bool = Field( - default=DEFAULT_CERT_TR_ENABLED, validation_alias="ENABLE_CERT_TR" - ) + enable_cert_tr: bool = Field(default=DEFAULT_CERT_TR_ENABLED, validation_alias="ENABLE_CERT_TR") proton_bridge_email: str | None = Field(default=None, validation_alias="PROTON_BRIDGE_EMAIL") proton_bridge_password: str | None = Field( default=None, validation_alias="PROTON_BRIDGE_PASSWORD" @@ -129,20 +127,14 @@ def __init__( default=DEFAULT_PROTON_FOLDER, validation_alias="PROTON_IMAP_FOLDER" ) # Multi-account overrides (comma/space separated) - proton_bridge_emails: str | None = Field( - default=None, validation_alias="PROTON_BRIDGE_EMAILS" - ) + proton_bridge_emails: str | None = Field(default=None, validation_alias="PROTON_BRIDGE_EMAILS") proton_bridge_passwords: str | None = Field( default=None, validation_alias="PROTON_BRIDGE_PASSWORDS" ) - proton_imap_folders: str | None = Field( - default=None, validation_alias="PROTON_IMAP_FOLDERS" - ) + proton_imap_folders: str | None = Field(default=None, validation_alias="PROTON_IMAP_FOLDERS") # Gmail (alternative IMAP for CERT-TR, e.g., imap.gmail.com) gmail_email: str | None = Field(default=None, validation_alias="GMAIL_EMAIL") - gmail_app_password: str | None = Field( - default=None, validation_alias="GMAIL_APP_PASSWORD" - ) + gmail_app_password: str | None = Field(default=None, validation_alias="GMAIL_APP_PASSWORD") gmail_password: str | None = Field(default=None, validation_alias="GMAIL_PASSWORD") gmail_imap_host: str = Field( default=DEFAULT_GMAIL_IMAP_HOST, validation_alias="GMAIL_IMAP_HOST" @@ -157,12 +149,8 @@ def __init__( default=DEFAULT_GMAIL_FOLDER, validation_alias="GMAIL_IMAP_FOLDER" ) gmail_emails: str | None = Field(default=None, validation_alias="GMAIL_EMAILS") - gmail_app_passwords: str | None = Field( - default=None, validation_alias="GMAIL_APP_PASSWORDS" - ) - gmail_imap_folders: str | None = Field( - default=None, validation_alias="GMAIL_IMAP_FOLDERS" - ) + gmail_app_passwords: str | None = Field(default=None, validation_alias="GMAIL_APP_PASSWORDS") + gmail_imap_folders: str | None = Field(default=None, validation_alias="GMAIL_IMAP_FOLDERS") # Gmail OAuth (alternative to App Password, per-account refresh token) gmail_oauth_client_id: str | None = Field( default=None, validation_alias="GMAIL_OAUTH_CLIENT_ID" @@ -185,9 +173,7 @@ def __init__( validation_alias="CERT_TR_SENDER_ALLOWLIST", ) cert_tr_max_mails: int = Field(default=200, validation_alias="CERT_TR_MAX_MAILS") - cert_tr_search_days: int | None = Field( - default=None, validation_alias="CERT_TR_SEARCH_DAYS" - ) + cert_tr_search_days: int | None = Field(default=None, validation_alias="CERT_TR_SEARCH_DAYS") def extra_repo_list(self) -> list[str]: import logging @@ -494,7 +480,9 @@ def get_proton_accounts(self) -> list[dict[str, str]]: # Align lengths: pad passwords/folders with defaults out: list[dict[str, str]] = [] - default_folder = (self.proton_imap_folder or DEFAULT_PROTON_FOLDER).strip() or DEFAULT_PROTON_FOLDER + default_folder = ( + self.proton_imap_folder or DEFAULT_PROTON_FOLDER + ).strip() or DEFAULT_PROTON_FOLDER # If single password provided but multiple emails, reuse it if len(passwords) == 1 and len(emails) > 1: passwords = passwords * len(emails) @@ -511,7 +499,9 @@ def get_proton_accounts(self) -> list[dict[str, str]]: "folder": folder, "host": (self.proton_bridge_host or DEFAULT_PROTON_IMAP_HOST).strip(), "port": str(self.proton_imap_port), - "security": (self.proton_imap_security or DEFAULT_PROTON_IMAP_SECURITY).strip().upper(), + "security": (self.proton_imap_security or DEFAULT_PROTON_IMAP_SECURITY) + .strip() + .upper(), } ) return out @@ -537,7 +527,9 @@ def get_proton_accounts(self) -> list[dict[str, str]]: "folder": folder, "host": (self.proton_bridge_host or DEFAULT_PROTON_IMAP_HOST).strip(), "port": str(self.proton_imap_port), - "security": (self.proton_imap_security or DEFAULT_PROTON_IMAP_SECURITY).strip().upper(), + "security": (self.proton_imap_security or DEFAULT_PROTON_IMAP_SECURITY) + .strip() + .upper(), } ) return out @@ -552,6 +544,7 @@ def _load_gmail_oauth_file_tokens(self) -> dict[str, object]: import json import logging from pathlib import Path + try: path = Path(self.gmail_oauth_token_file) if not path.exists(): @@ -559,7 +552,9 @@ def _load_gmail_oauth_file_tokens(self) -> dict[str, object]: data = json.loads(path.read_text(encoding="utf-8")) return data if isinstance(data, dict) else {} except (OSError, ValueError, json.JSONDecodeError) as e: - logging.getLogger(__name__).debug("Failed to load Gmail OAuth file %s: %s", self.gmail_oauth_token_file, e) + logging.getLogger(__name__).debug( + "Failed to load Gmail OAuth file %s: %s", self.gmail_oauth_token_file, e + ) return {} def get_gmail_accounts(self) -> list[dict[str, str]]: @@ -658,7 +653,9 @@ def _split_passwords(raw: str | None) -> list[str]: file_map[k.lower()] = str(v["refresh_token"]) # Also consider env file client_id/secret for file-based accounts out: list[dict[str, str]] = [] - default_folder = (self.gmail_imap_folder or DEFAULT_GMAIL_FOLDER).strip() or DEFAULT_GMAIL_FOLDER + default_folder = ( + self.gmail_imap_folder or DEFAULT_GMAIL_FOLDER + ).strip() or DEFAULT_GMAIL_FOLDER if len(passwords) == 1 and len(emails) > 1: passwords = passwords * len(emails) if len(oauth_refresh_tokens) == 1 and len(emails) > 1: @@ -666,7 +663,11 @@ def _split_passwords(raw: str | None) -> list[str]: # Handle single email with multiple folders: expand to multiple accounts if len(emails) == 1 and len(folders) > 1: single_email = emails[0] - single_pw = passwords[0] if passwords else (self.gmail_app_password or self.gmail_password or "") + single_pw = ( + passwords[0] + if passwords + else (self.gmail_app_password or self.gmail_password or "") + ) single_oauth = oauth_refresh_tokens[0] if oauth_refresh_tokens else "" for folder in folders: folder = folder.strip() or default_folder @@ -681,7 +682,9 @@ def _split_passwords(raw: str | None) -> list[str]: "folder": folder, "host": (self.gmail_imap_host or DEFAULT_GMAIL_IMAP_HOST).strip(), "port": str(self.gmail_imap_port), - "security": (self.gmail_imap_security or DEFAULT_GMAIL_IMAP_SECURITY).strip().upper(), + "security": (self.gmail_imap_security or DEFAULT_GMAIL_IMAP_SECURITY) + .strip() + .upper(), "provider": "gmail", "auth_method": auth_method, } @@ -691,7 +694,11 @@ def _split_passwords(raw: str | None) -> list[str]: csec = self.gmail_oauth_client_secret or "" if not cid and isinstance(file_tokens, dict) and file_tokens.get("client_id"): cid = str(file_tokens["client_id"]) - if not csec and isinstance(file_tokens, dict) and file_tokens.get("client_secret"): + if ( + not csec + and isinstance(file_tokens, dict) + and file_tokens.get("client_secret") + ): csec = str(file_tokens["client_secret"]) acct["oauth_client_id"] = (cid or "").strip() acct["oauth_client_secret"] = (csec or "").strip() @@ -730,7 +737,9 @@ def _split_passwords(raw: str | None) -> list[str]: "folder": folder, "host": (self.gmail_imap_host or DEFAULT_GMAIL_IMAP_HOST).strip(), "port": str(self.gmail_imap_port), - "security": (self.gmail_imap_security or DEFAULT_GMAIL_IMAP_SECURITY).strip().upper(), + "security": (self.gmail_imap_security or DEFAULT_GMAIL_IMAP_SECURITY) + .strip() + .upper(), "provider": "gmail", "auth_method": auth_method, } @@ -766,7 +775,9 @@ def get_cert_tr_accounts(self) -> list[dict[str, str]]: def cert_tr_sender_list(self) -> list[str]: if not self.cert_tr_sender_allowlist or not self.cert_tr_sender_allowlist.strip(): return [DEFAULT_CERT_TR_SENDER_ALLOWLIST] - return [s.strip().lower() for s in self._split_list(self.cert_tr_sender_allowlist) if s.strip()] + return [ + s.strip().lower() for s in self._split_list(self.cert_tr_sender_allowlist) if s.strip() + ] def effective_bind_address(self) -> str: if self.host and self.bind_address == DEFAULT_BIND_ADDRESS: diff --git a/src/advisory_rss/logging_config.py b/src/advisory_rss/logging_config.py index bd5b1d2..597cf32 100644 --- a/src/advisory_rss/logging_config.py +++ b/src/advisory_rss/logging_config.py @@ -289,6 +289,7 @@ def setup_logging( _configured_format = fmt_str _configured_log_file = normalized_log_file or "" + def get_uvicorn_log_config(level: str = "INFO") -> dict[str, Any]: lvl = (level or "INFO").upper() if lvl not in ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"): diff --git a/src/advisory_rss/rss/builder.py b/src/advisory_rss/rss/builder.py index 255af27..c114f92 100644 --- a/src/advisory_rss/rss/builder.py +++ b/src/advisory_rss/rss/builder.py @@ -78,10 +78,16 @@ def _build_item_xml(adv: NormalizedAdvisory) -> str: raw_link = adv.html_url.strip() if adv.html_url else "" if src == "cert-tr": # fallback to NVD or siberguvenlik portal if not safe - fallback = f"https://nvd.nist.gov/vuln/detail/{adv.cve_id}" if adv.cve_id else "https://siberguvenlik.gov.tr" + fallback = ( + f"https://nvd.nist.gov/vuln/detail/{adv.cve_id}" + if adv.cve_id + else "https://siberguvenlik.gov.tr" + ) link = raw_link if _is_safe_url(raw_link) else fallback else: - link = raw_link if _is_safe_url(raw_link) else f"https://github.com/advisories/{adv.ghsa_id}" + link = ( + raw_link if _is_safe_url(raw_link) else f"https://github.com/advisories/{adv.ghsa_id}" + ) link_esc = escape_text(link) guid = escape_text(adv.ghsa_id) @@ -262,7 +268,9 @@ def _build_content_html(adv: NormalizedAdvisory) -> str: # source footer src = getattr(adv, "source", "github") or "github" if src == "cert-tr": - parts.append('

Source: CERT-TR / Siber Güvenlik Başkanlığı (via Proton Mail)

') + parts.append( + "

Source: CERT-TR / Siber Güvenlik Başkanlığı (via Proton Mail)

" + ) html_str = "\n".join(parts) # Bound content size per item diff --git a/src/advisory_rss/server/app.py b/src/advisory_rss/server/app.py index d0cc80b..6551593 100644 --- a/src/advisory_rss/server/app.py +++ b/src/advisory_rss/server/app.py @@ -115,7 +115,9 @@ async def health() -> JSONResponse: # per-source counts try: all_advs = cache.load_all() - github_count = sum(1 for a in all_advs if (getattr(a, "source", "github") or "github") == "github") + github_count = sum( + 1 for a in all_advs if (getattr(a, "source", "github") or "github") == "github" + ) cert_tr_count = sum(1 for a in all_advs if getattr(a, "source", "") == "cert-tr") except (OSError, ValueError, RuntimeError, AttributeError) as e: logger.debug("Failed to load per-source counts: %s", e) @@ -138,7 +140,9 @@ async def health() -> JSONResponse: "github_count": github_count, "cert_tr_count": cert_tr_count, "cert_tr_enabled": settings.enable_cert_tr, - "cert_tr_accounts": len(settings.get_cert_tr_accounts()) if settings.enable_cert_tr else 0, + "cert_tr_accounts": len(settings.get_cert_tr_accounts()) + if settings.enable_cert_tr + else 0, "cert_tr_diag": cert_tr_diag, "last_successful_sync": meta.last_successful_sync, "next_scheduled_sync": meta.next_scheduled_sync, @@ -179,11 +183,15 @@ async def rss(request: Request) -> Response: # Serve from cache only - never call GitHub/IMAP here advisories = cache.load_sorted(limit=None) # load all sorted, builder caps by max_items if src_filter != "all": - advisories = [a for a in advisories if (getattr(a, "source", "github") or "github") == src_filter] + advisories = [ + a for a in advisories if (getattr(a, "source", "github") or "github") == src_filter + ] feed_user = cache.get_meta("authenticated_user") or "user" ttl_minutes = max(1, settings.refresh_interval // 60) # Mixed feed title when both sources present - has_github = any((getattr(a, "source", "github") or "github") == "github" for a in advisories) + has_github = any( + (getattr(a, "source", "github") or "github") == "github" for a in advisories + ) has_cert = any(getattr(a, "source", "") == "cert-tr" for a in advisories) if has_github and has_cert and src_filter == "all": feed_title = f"Security Advisories (GitHub + CERT-TR) - @{feed_user}" @@ -195,7 +203,11 @@ async def rss(request: Request) -> Response: feed_title = f"GitHub Security Advisories - @{feed_user}" feed_desc = f"Security advisories created by {feed_user} (author={feed_user}) via GitHub API - localhost-only feed" # Keep feed_link sensible - feed_link = f"https://github.com/{feed_user}" if src_filter != "cert-tr" else "https://siberguvenlik.gov.tr" + feed_link = ( + f"https://github.com/{feed_user}" + if src_filter != "cert-tr" + else "https://siberguvenlik.gov.tr" + ) xml = build_rss( advisories, feed_title=feed_title, @@ -294,7 +306,10 @@ async def refresh(request: Request) -> JSONResponse: diag_comb["cert_tr"] = diag_ct diag_comb["errors"].extend(diag_ct.get("errors") or []) if not all_advs: - raise HTTPException(status_code=502, detail=f"Refresh: no advisories (errors={diag_comb['errors'][:2]})") + raise HTTPException( + status_code=502, + detail=f"Refresh: no advisories (errors={diag_comb['errors'][:2]})", + ) count = cache.upsert_advisories(all_advs) # keep user from GitHub if present user = None @@ -307,7 +322,9 @@ async def refresh(request: Request) -> JSONResponse: import json if diag_comb.get("cert_tr"): - cache.set_meta("cert_tr_diag", json.dumps(diag_comb["cert_tr"], ensure_ascii=False)) + cache.set_meta( + "cert_tr_diag", json.dumps(diag_comb["cert_tr"], ensure_ascii=False) + ) except (OSError, ValueError, TypeError, RuntimeError) as e: logger.debug("Failed to persist cert_tr_diag: %s", e) logger.info("POST /refresh ok: count=%d", count, extra={"diag": diag_comb}) @@ -367,7 +384,9 @@ async def background_refresh_loop(settings: Settings, cache: CacheStore) -> None continue # Also skip if CERT-TR enabled but no IMAP accounts and no token => nothing to do if not token and settings.enable_cert_tr and not settings.get_cert_tr_accounts(): - logger.info("Background refresh skipped - CERT-TR enabled but no IMAP accounts and no GitHub token") + logger.info( + "Background refresh skipped - CERT-TR enabled but no IMAP accounts and no GitHub token" + ) continue try: all_advs = [] diff --git a/tests/unit/test_cert_tr_imap.py b/tests/unit/test_cert_tr_imap.py index fea713c..310fba8 100644 --- a/tests/unit/test_cert_tr_imap.py +++ b/tests/unit/test_cert_tr_imap.py @@ -3,7 +3,10 @@ from advisory_rss.cert_tr.imap import fetch_raw_emails -SAMPLE_RAW = b"From: cve@siberguvenlik.gov.tr\r\nSubject: Test CVE-2026-0001\r\n\r\nBody CVE-2026-0001" +SAMPLE_RAW = ( + b"From: cve@siberguvenlik.gov.tr\r\nSubject: Test CVE-2026-0001\r\n\r\nBody CVE-2026-0001" +) + def _mock_imap(data_bytes_list): """Create mock IMAP object that returns uids and fetches.""" @@ -12,6 +15,7 @@ def _mock_imap(data_bytes_list): mock.select.return_value = ("OK", [b"1"]) # search returns uids mock.search.return_value = ("OK", [b"1 2"]) + # fetch returns tuple per uid def fetch_side(uid, _what): # uid can be bytes like b"2" or b"1" @@ -23,11 +27,20 @@ def fetch_side(uid, _what): idx = 0 if uid_s == "2" else 1 raw = data_bytes_list[idx] if idx < len(data_bytes_list) else data_bytes_list[0] return ("OK", [(b"1 (BODY[] {10}", raw)]) + mock.fetch.side_effect = fetch_side return mock + def test_fetch_never_marks_read(): - acc = {"email": "a@proton.me", "password": "p", "folder": "INBOX", "host": "127.0.0.1", "port": "1143", "security": "STARTTLS"} + acc = { + "email": "a@proton.me", + "password": "p", + "folder": "INBOX", + "host": "127.0.0.1", + "port": "1143", + "security": "STARTTLS", + } with patch("advisory_rss.cert_tr.imap.connect_imap") as mock_connect: mock = _mock_imap([SAMPLE_RAW, SAMPLE_RAW]) mock_connect.return_value = mock @@ -44,9 +57,18 @@ def test_fetch_never_marks_read(): assert kwargs.get("readonly") is True assert len(raws) == 2 + def test_fetch_multi_account_distinct_folders(): from advisory_rss.config.settings import Settings - s = Settings(_env_file="/dev/null", ENABLE_CERT_TR="true", PROTON_BRIDGE_EMAILS="a@proton.me,b@proton.me", PROTON_BRIDGE_PASSWORDS="p1,p2", PROTON_IMAP_FOLDERS="INBOX,INBOX.CERT-TR", _env_file_encoding="utf-8") + + s = Settings( + _env_file="/dev/null", + ENABLE_CERT_TR="true", + PROTON_BRIDGE_EMAILS="a@proton.me,b@proton.me", + PROTON_BRIDGE_PASSWORDS="p1,p2", + PROTON_IMAP_FOLDERS="INBOX,INBOX.CERT-TR", + _env_file_encoding="utf-8", + ) accs = s.get_proton_accounts() assert accs[0]["folder"] == "INBOX" assert accs[1]["folder"] == "INBOX.CERT-TR" diff --git a/tests/unit/test_cert_tr_parser.py b/tests/unit/test_cert_tr_parser.py index 6cdd1d3..7d126dc 100644 --- a/tests/unit/test_cert_tr_parser.py +++ b/tests/unit/test_cert_tr_parser.py @@ -20,7 +20,9 @@ def test_cert_tr_parse_sample(): def test_cert_tr_sender_allowlist(): - assert is_cert_tr_sender("Ürün Güvenliği Koordinasyon Ekibi ", ["siberguvenlik.gov.tr"]) + assert is_cert_tr_sender( + "Ürün Güvenliği Koordinasyon Ekibi ", ["siberguvenlik.gov.tr"] + ) assert is_cert_tr_sender("cve@siberguvenlik.gov.tr", ["siberguvenlik.gov.tr"]) assert not is_cert_tr_sender("noreply@github.com", ["siberguvenlik.gov.tr"]) assert is_cert_tr_sender("test@usom.gov.tr", ["siberguvenlik.gov.tr", "usom.gov.tr"]) @@ -44,7 +46,9 @@ def test_cert_tr_parser_handles_quoted_printable_charset(): adv = parse_cert_tr_email(raw) assert adv is not None # Check Turkish chars decoded correctly - assert "TÜBİTAK" in adv.description or "TUBITAK" in adv.description or "BİLGEM" in adv.description + assert ( + "TÜBİTAK" in adv.description or "TUBITAK" in adv.description or "BİLGEM" in adv.description + ) def test_cert_tr_parser_no_cve_skipped(): diff --git a/tests/unit/test_cert_tr_rss.py b/tests/unit/test_cert_tr_rss.py index bc6031e..4aa9195 100644 --- a/tests/unit/test_cert_tr_rss.py +++ b/tests/unit/test_cert_tr_rss.py @@ -29,7 +29,15 @@ def test_mixed_rss(): package_name="pardus-etap-settings", identifiers=[{"type": "CWE", "value": "CWE-862"}], ) - xml = build_rss([gh, ct], feed_title="T", feed_link="https://example.com", feed_description="d", max_items=100, ttl_minutes=10, authenticated_user="user") + xml = build_rss( + [gh, ct], + feed_title="T", + feed_link="https://example.com", + feed_description="d", + max_items=100, + ttl_minutes=10, + authenticated_user="user", + ) assert "source:github" in xml assert "source:cert-tr" in xml assert "[CERT-TR]" in xml @@ -37,11 +45,24 @@ def test_mixed_rss(): # order: GHSA newer first if sorted inside builder assert xml.index("GHSA-xxxx") < xml.index("CERT-TR-CVE") + def test_cert_tr_rss_filter(): - gh = NormalizedAdvisory(ghsa_id="GHSA-1", summary="g", source="github", updated_at=datetime(2026,9,20,tzinfo=UTC), published_at=datetime(2026,9,20,tzinfo=UTC)) - ct = NormalizedAdvisory(ghsa_id="CERT-TR-CVE-1", summary="c", source="cert-tr", updated_at=datetime(2026,9,15,tzinfo=UTC), published_at=datetime(2026,9,15,tzinfo=UTC)) + gh = NormalizedAdvisory( + ghsa_id="GHSA-1", + summary="g", + source="github", + updated_at=datetime(2026, 9, 20, tzinfo=UTC), + published_at=datetime(2026, 9, 20, tzinfo=UTC), + ) + ct = NormalizedAdvisory( + ghsa_id="CERT-TR-CVE-1", + summary="c", + source="cert-tr", + updated_at=datetime(2026, 9, 15, tzinfo=UTC), + published_at=datetime(2026, 9, 15, tzinfo=UTC), + ) # simulate server filter all_adv = [gh, ct] - filtered = [a for a in all_adv if getattr(a,"source","github")=="cert-tr"] - assert len(filtered)==1 - assert filtered[0].ghsa_id=="CERT-TR-CVE-1" + filtered = [a for a in all_adv if getattr(a, "source", "github") == "cert-tr"] + assert len(filtered) == 1 + assert filtered[0].ghsa_id == "CERT-TR-CVE-1" diff --git a/tests/unit/test_cert_tr_settings.py b/tests/unit/test_cert_tr_settings.py index 6a397ca..84b0052 100644 --- a/tests/unit/test_cert_tr_settings.py +++ b/tests/unit/test_cert_tr_settings.py @@ -2,7 +2,14 @@ def test_single_account(): - s = Settings(_env_file="/dev/null", ENABLE_CERT_TR="true", PROTON_BRIDGE_EMAIL="a@proton.me", PROTON_BRIDGE_PASSWORD="p1", PROTON_IMAP_FOLDER="INBOX", _env_file_encoding="utf-8") + s = Settings( + _env_file="/dev/null", + ENABLE_CERT_TR="true", + PROTON_BRIDGE_EMAIL="a@proton.me", + PROTON_BRIDGE_PASSWORD="p1", + PROTON_IMAP_FOLDER="INBOX", + _env_file_encoding="utf-8", + ) accs = s.get_proton_accounts() assert len(accs) == 1 assert accs[0]["email"] == "a@proton.me" @@ -42,12 +49,18 @@ def test_multi_account_fallback_single_password(): def test_folder_validation(): - s = Settings(_env_file="/dev/null", PROTON_IMAP_FOLDER="INBOX.CERT-TR", _env_file_encoding="utf-8") + s = Settings( + _env_file="/dev/null", PROTON_IMAP_FOLDER="INBOX.CERT-TR", _env_file_encoding="utf-8" + ) assert s.proton_imap_folder == "INBOX.CERT-TR" def test_sender_allowlist(): - s = Settings(_env_file="/dev/null", CERT_TR_SENDER_ALLOWLIST="siberguvenlik.gov.tr, usom.gov.tr", _env_file_encoding="utf-8") + s = Settings( + _env_file="/dev/null", + CERT_TR_SENDER_ALLOWLIST="siberguvenlik.gov.tr, usom.gov.tr", + _env_file_encoding="utf-8", + ) assert "siberguvenlik.gov.tr" in s.cert_tr_sender_list() assert "usom.gov.tr" in s.cert_tr_sender_list() diff --git a/tests/unit/test_normalize.py b/tests/unit/test_normalize.py index 221e37e..0ec6ef4 100644 --- a/tests/unit/test_normalize.py +++ b/tests/unit/test_normalize.py @@ -74,7 +74,7 @@ def test_normalize_full(): assert adv.cve_id == raw.get("cve_id") assert adv.severity == (raw.get("severity") or "medium").lower() assert adv.summary == raw.get("summary") - # author is per-user - compare to fixture, not hardcoded Lunixizm0 + expected_author = ( (raw.get("author") or {}).get("login") if isinstance(raw.get("author"), dict) else None )