diff --git a/app/adapters/http.py b/app/adapters/http.py index 0cc1c19..35efc0c 100644 --- a/app/adapters/http.py +++ b/app/adapters/http.py @@ -4,13 +4,16 @@ redirect hop before connecting. """ +import io import ipaddress import socket import time +import warnings from urllib.parse import urljoin, urlparse import requests from bs4 import BeautifulSoup +from PIL import Image, UnidentifiedImageError from requests.exceptions import RequestException from app.domain.post import is_safe_url @@ -242,19 +245,23 @@ def _make_robust_request(url: str) -> requests.Response | None: # One short attempt per image: a static CDN either answers at once or the image # is not worth holding the run for (the summarizer also caps the total). IMAGE_PROBE_TIMEOUT_SECONDS = 10 +# Enough of the file for PNG, GIF and JPEG headers (a JPEG's size marker can sit +# behind tens of KB of EXIF data); the rest of the image is never downloaded. +IMAGE_HEADER_BYTES = 64 * 1024 -def is_email_renderable_image(url: str) -> bool: - """Whether ``url`` serves an image a mail client will display. +def probe_email_image(url: str) -> int | None: + """The pixel width of the image at ``url`` if a mail client will display + it (0 when the width cannot be read), else None. Fetched, not guessed from the URL: CDN URLs often carry no extension, and the only proof an image loads is that it does. The request asks for the email-safe types, so a content-negotiating CDN answers with one of them if - it can. Only the headers are read. SSRF-checked at every redirect hop like - every other fetch. + it can. Only the first bytes are read. SSRF-checked at every redirect hop + like every other fetch. """ if not is_safe_url(url): - return False + return None headers = REQUEST_HEADERS | { "Accept": "image/png,image/jpeg,image/gif;q=0.9,*/*;q=0.1" } @@ -268,11 +275,42 @@ def is_email_renderable_image(url: str) -> bool: ) except RequestException as e: logger.info("Image probe failed for '%s': %s", url, e) - return False + return None if response is None: - return False + return None with response: if not response.ok: - return False + return None content_type = response.headers.get("content-type", "").split(";")[0] - return content_type.strip().lower() in EMAIL_IMAGE_TYPES + if content_type.strip().lower() not in EMAIL_IMAGE_TYPES: + return None + return _image_width(response, time.monotonic() + IMAGE_PROBE_TIMEOUT_SECONDS) + + +def _image_width(response: requests.Response, deadline: float) -> int: + """The width from the image's header bytes, or 0 if they do not say. + + Only the header is parsed (``.size``), never the pixels. A header claiming a + huge image trips Pillow's decompression-bomb guard, which is still a + displayable image of unknown width, not a reason to lose the card. + """ + head = b"" + try: + for chunk in response.iter_content(chunk_size=16 * 1024): + head += chunk + # The timeout bounds each read, not the total: a host that drips + # bytes would otherwise hold the probe indefinitely. + if len(head) >= IMAGE_HEADER_BYTES or time.monotonic() >= deadline: + break + with warnings.catch_warnings(): + warnings.simplefilter("ignore", Image.DecompressionBombWarning) + with Image.open(io.BytesIO(head)) as image: + return int(image.size[0]) + except ( + RequestException, + UnidentifiedImageError, + Image.DecompressionBombError, + OSError, + ValueError, + ): + return 0 diff --git a/app/application/ports.py b/app/application/ports.py index 4ee8bf5..4699524 100644 --- a/app/application/ports.py +++ b/app/application/ports.py @@ -113,5 +113,6 @@ def is_converse(self, model: Any) -> bool: differs from the native request body's.""" -#: Whether an image URL serves a type every mail client displays. -ImageProbe = Callable[[str], bool] +#: The pixel width of a mail-displayable image at a URL (0 if unknown), or None +#: if mail clients would not display it. +ImageProbe = Callable[[str], int | None] diff --git a/app/application/prompts/__init__.py b/app/application/prompts/__init__.py index 15c29f4..d514e2b 100644 --- a/app/application/prompts/__init__.py +++ b/app/application/prompts/__init__.py @@ -4,6 +4,7 @@ GreetingPrompt, GreetingRevisionPrompt, SummarizationPrompt, + SummaryLengthPrompt, TitleCasePrompt, ) @@ -13,5 +14,6 @@ "GreetingPrompt", "GreetingRevisionPrompt", "SummarizationPrompt", + "SummaryLengthPrompt", "TitleCasePrompt", ] diff --git a/app/application/prompts/prompts.py b/app/application/prompts/prompts.py index 1a5e4f6..dc4cd43 100644 --- a/app/application/prompts/prompts.py +++ b/app/application/prompts/prompts.py @@ -428,8 +428,8 @@ class GreetingRevisionPrompt(BasePrompt): {draft} Count the words rather than estimating. If it is too long, start by dropping a whole sentence -(three sentences including the opener are enough). Keep the opener "Hey friends! I'm Peccy ๐Ÿ˜Ž", -the specific observation and the hand-off to the articles, and keep the casual confident tone. +(three sentences including the opener are enough). Keep the opener "Hey friends! I'm Peccy ๐Ÿ˜Ž" +and the specific observation, and keep the casual confident tone. Do NOT introduce a "back then X, now Y" contrast. Plain text only. **OUTPUT:** The revised introduction only.""", @@ -440,8 +440,8 @@ class GreetingRevisionPrompt(BasePrompt): {draft} ๊ธ€์ž ์ˆ˜๋Š” ์–ด๋ฆผํ•˜์ง€ ๋ง๊ณ  ์„ธ์„ธ์š”. ๋„ˆ๋ฌด ๊ธธ๋ฉด ๋ฌธ์žฅ ํ•˜๋‚˜๋ฅผ ํ†ต์งธ๋กœ ๋นผ๋Š” ๊ฒƒ๋ถ€ํ„ฐ ํ•˜์„ธ์š”(์—ฌ๋Š” ์ธ์‚ฌ -ํฌํ•จ 3๋ฌธ์žฅ์ด๋ฉด ์ถฉ๋ถ„ํ•ฉ๋‹ˆ๋‹ค). ์—ฌ๋Š” ์ธ์‚ฌ "์•ˆ๋…• ์นœ๊ตฌ๋“ค! ๋‚œ Peccy์•ผ ๐Ÿ˜Ž"์™€ ๊ตฌ์ฒด์ ์ธ ๊ด€์ฐฐ, ์•„ํ‹ฐํด๋กœ -๋„˜๊ธฐ๋Š” ๋งˆ์ง€๋ง‰ ๋ฌธ์žฅ์€ ์œ ์ง€ํ•˜๊ณ , ๋ฐ˜๋งยท์ž์‹ ๊ฐ ์žˆ๋Š” ์–ด์กฐ๋„ ๊ทธ๋Œ€๋กœ ๋‘์„ธ์š”. "์˜ˆ์ „์—” X์˜€๋Š”๋ฐ ์ด์ œ +ํฌํ•จ 3๋ฌธ์žฅ์ด๋ฉด ์ถฉ๋ถ„ํ•ฉ๋‹ˆ๋‹ค). ์—ฌ๋Š” ์ธ์‚ฌ "์•ˆ๋…• ์นœ๊ตฌ๋“ค! ๋‚œ Peccy์•ผ ๐Ÿ˜Ž"์™€ ๊ตฌ์ฒด์ ์ธ ๊ด€์ฐฐ์€ ์œ ์ง€ํ•˜๊ณ , +๋ฐ˜๋งยท์ž์‹ ๊ฐ ์žˆ๋Š” ์–ด์กฐ๋„ ๊ทธ๋Œ€๋กœ ๋‘์„ธ์š”. "์˜ˆ์ „์—” X์˜€๋Š”๋ฐ ์ด์ œ Y์•ผ" ๊ฐ™์€ ๊ณผ๊ฑฐ-ํ˜„์žฌ ๋Œ€์กฐ๋Š” ๋„ฃ์ง€ ๋งˆ์„ธ์š”. ํ‰๋ฌธ๋งŒ ์ถœ๋ ฅํ•˜์„ธ์š”. **OUTPUT:** ์ˆ˜์ •๋œ ์ธํŠธ๋กœ๋งŒ ์ถœ๋ ฅ.""", @@ -883,3 +883,54 @@ def for_language( }, ) return prompt_class + + +class SummaryLengthPrompt(BasePrompt): + """Ask the summarization model to bring an over-long summary under its budget. + + The budget is stated in the summarization prompt, and the model still runs a + long, dense article a quarter over it. Length is something code measures + exactly, so an over-long summary is sent back once with the measured count. + """ + + input_variables: list[str] = ["summary", "measured", "limit"] + output_variables: list[str] = ["summary"] + system_prompt_template: str = SummarizationPrompt.system_prompt_template + human_prompt_template: str = "" + + _human_prompt_template: ClassVar[dict[Language, str]] = { + Language.EN: """Your summary below is {measured} words of visible text. The limit is +{limit} words, so shorten it to fit. + +{summary} + +Cut sentences that restate something already said, background the reader does not need, and +the least important details first. Keep every measured number, parameter name, code block and +table, the sections and their order, and the HTML markup and writing style the draft +already follows. Add nothing new. + +**OUTPUT:** [the shortened summary] only.""", + Language.KO: """์•„๋ž˜ ์š”์•ฝ์€ ๊ฐ€์‹œ ํ…์ŠคํŠธ ๊ธฐ์ค€ {measured}์ž์ž…๋‹ˆ๋‹ค. ์ƒํ•œ์ด {limit}์ž์ด๋‹ˆ ๊ทธ ์•ˆ์œผ๋กœ +์ค„์ด์„ธ์š”. + +{summary} + +์•ž์—์„œ ํ•œ ๋ง์„ ๋˜ํ’€์ดํ•œ ๋ฌธ์žฅ, ๋…์ž์—๊ฒŒ ํ•„์š” ์—†๋Š” ๋ฐฐ๊ฒฝ, ๋œ ์ค‘์š”ํ•œ ์„ธ๋ถ€๋ถ€ํ„ฐ ์ง€์šฐ์„ธ์š”. ์ธก์ •๊ฐ’, +ํŒŒ๋ผ๋ฏธํ„ฐ ์ด๋ฆ„, ์ฝ”๋“œ ๋ธ”๋ก, ํ‘œ๋Š” ์ง€์šฐ์ง€ ๋ง๊ณ  ์„น์…˜ ๊ตฌ์„ฑ๊ณผ ์ˆœ์„œ, HTML ๋งˆํฌ์—…๊ณผ ์ดˆ์•ˆ์˜ ๋ฌธ์ฒด(ํ•ฉ๋‹ˆ๋‹ค์ฒด, +๋ฒˆ์—ญํˆฌยท์ƒํˆฌ๊ตฌ ์—†์ด)๋„ ๊ทธ๋Œ€๋กœ ๋‘์„ธ์š”. ์ƒˆ ๋‚ด์šฉ์€ ๋”ํ•˜์ง€ ๋งˆ์„ธ์š”. + +**OUTPUT:** [์ค„์ธ ์š”์•ฝ]๋งŒ ์ถœ๋ ฅ.""", + } + + @classmethod + def for_language( + cls, language: Language = Language.KO + ) -> type["SummaryLengthPrompt"]: + return type( + f"{language.name.capitalize()}SummaryLengthPrompt", + (cls,), + { + "system_prompt_template": cls.system_prompt_template, + "human_prompt_template": cls._human_prompt_template[language], + }, + ) diff --git a/app/application/summarize.py b/app/application/summarize.py index 8753d13..7e7de22 100644 --- a/app/application/summarize.py +++ b/app/application/summarize.py @@ -1,3 +1,4 @@ +import functools import time from collections.abc import Callable from typing import Any, ClassVar @@ -11,9 +12,14 @@ BatchProcessor, HTMLTagOutputParser, measure_execution_time, + retry_with_backoff, ) from app.application.ports import ChatModelProvider, ImageProbe -from app.application.prompts.prompts import FilteringPrompt, SummarizationPrompt +from app.application.prompts.prompts import ( + FilteringPrompt, + SummarizationPrompt, + SummaryLengthPrompt, +) from app.domain.constants import Language, LanguageModelId from app.domain.content import content_html, visible_text from app.domain.eval_metrics import is_anchor @@ -25,6 +31,7 @@ SUMMARY_FAILED, SUMMARY_INVALID, SummarizationOptions, + SummarizerConfig, SummaryOutput, _encode_image_srcs, _image_candidates, @@ -48,14 +55,15 @@ def _drop_unrenderable_images( budget_seconds: float = IMAGE_PROBE_BUDGET_SECONDS, clock: Callable[[], float] = time.monotonic, ) -> tuple[str, list[str]]: - """Keep only images a mail client will display. + """Keep only images a mail client will display, sized to their own width. Outlook for Windows shows a WebP image's alt text instead, so the alt text and the caption ran together as one odd sentence ("Massive Text Embedding Benchmark (Code)์ฝ”๋“œ ๊ฒ€์ƒ‰ ๋ถ€๋ฌธ ... ์ฐจํŠธ์ž…๋‹ˆ๋‹ค"). An image that fails is swapped for another rendition the article lists (NVIDIA serves WebP as the - main file and PNG in its srcset); with none, its figure goes. Returns - ``(cleaned_html, dropped_srcs)``. + main file and PNG in its srcset); with none, its figure goes. A kept image + narrower than the column keeps its own width instead of being stretched to + it. Returns ``(cleaned_html, dropped_srcs)``. """ if "= deadline: break - if is_renderable(candidate): - chosen = candidate + if (probed := is_renderable(candidate)) is not None: + chosen, width = candidate, probed break if chosen is None: dropped.append(src) (img.find_parent("figure") or img).decompose() - elif chosen != src: + continue + if chosen != src: img["src"] = chosen changed = True + if 0 < width < SummarizerConfig.SUMMARY_IMAGE_WIDTH: + img["width"] = str(width) + img["style"] = "max-width:100%;height:auto" + changed = True return (str(soup), dropped) if (dropped or changed) else (summary_html, []) @@ -104,7 +117,8 @@ def __init__( image_probe: ImageProbe, ) -> None: self.settings = settings - self.image_probe = image_probe + # Cached per run: a shortened rewrite probes the same images again. + self.image_probe = functools.cache(image_probe) self.language = settings.language self.filtered_out_posts: list[tuple[Post, str]] = [] self.batch_processor = BatchProcessor( @@ -251,6 +265,11 @@ def _create_summarizer( and self._cache_can_hit(), use_converse=self.llm_factory.is_converse(summarization_llm), ) + self.shortener = ( + SummaryLengthPrompt.for_language(self.language).get_prompt() + | summarization_llm + | HTMLTagOutputParser(tag_names=["summary"], required_tags=["summary"]) + ) return prompt | summarization_llm | output_parser @measure_execution_time @@ -509,7 +528,7 @@ def prepare_inputs(items: list[Post]) -> list[dict[str, Any]]: (post, f"{SUMMARY_FAILED} ({problem}).") ) continue - post.summary = finalized + post.summary = self._fit_length(post, summary_output, finalized) post.one_liner = summary_output.one_liner post.tags = summary_output.tags post.urls = summary_output.urls @@ -532,6 +551,67 @@ def prepare_inputs(items: list[Post]) -> list[dict[str, Any]]: self.filtered_out_posts.append((post, f"{SUMMARY_INVALID}: {e}")) return succeeded + # The budget's upper end (KO characters, EN words), and how far past it a + # summary may run before it is sent back once to be shortened. + LENGTH_LIMIT: ClassVar[dict[Language, int]] = { + Language.KO: KO_LENGTH_BAND[1], + Language.EN: 2000, + } + LENGTH_TOLERANCE: ClassVar[float] = 1.1 + + def _length(self, summary_html: str) -> int: + text = visible_text(summary_html) + return len(text) if self.language is Language.KO else len(text.split()) + + @retry_with_backoff("summary-length") + def _shorten(self, summary_html: str, measured: int) -> dict[str, Any]: + limit = self.LENGTH_LIMIT[self.language] + return self.shortener.invoke( + {"summary": summary_html, "measured": measured, "limit": limit} + ) + + def _fit_length(self, post: Post, output: SummaryOutput, finalized: str) -> str: + """``finalized``, or a shortened rewrite if it runs well past the budget. + + The rewrite is kept only if it is usable and shorter; any failure keeps + the original, since a long summary is a flaw and a lost one is not. + """ + measured = self._length(finalized) + limit = self.LENGTH_LIMIT[self.language] + if measured <= limit * self.LENGTH_TOLERANCE: + return finalized + try: + revised = SummaryOutput.model_validate( + {"summary": self._shorten(output.summary, measured)["summary"]} + ).summary + shortened = self._finalize_summary(post, revised) + except Exception as e: + logger.warning( + "Shortening the %d-unit summary of '%s' failed (%s); keeping it.", + measured, + post.title, + e, + ) + return finalized + revised_length = self._length(shortened) + if self._unusable_summary(shortened) or revised_length >= measured: + logger.warning( + "Shortened summary of '%s' is not usable or not shorter (%d -> %d); " + "keeping the original.", + post.title, + measured, + revised_length, + ) + return finalized + logger.info( + "Summary of '%s' shortened from %d to %d (limit %d).", + post.title, + measured, + revised_length, + limit, + ) + return shortened + # Summary tags read as plain text. Not one_liner: _clean_one_liner already # decodes it once, and a second decode turned "List<String>" into # "List" on the card. diff --git a/app/main.py b/app/main.py index 6f2b166..cb31e50 100644 --- a/app/main.py +++ b/app/main.py @@ -21,7 +21,7 @@ ) from app.adapters.bedrock import BedrockLanguageModelFactory from app.adapters.feeds import PostCollector -from app.adapters.http import is_email_renderable_image +from app.adapters.http import probe_email_image from app.adapters.rendering import NewsletterBuilder from app.application.alerts import ( maybe_send_clipped_newsletter_alert, @@ -83,7 +83,7 @@ def handler(event: dict[str, Any], context: Any) -> dict[str, int | str]: config, PostCollector.from_urls, models, - is_email_renderable_image, + probe_email_image, end_date_str=end_date_str, language=language, ) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index dab40ee..a40b8e5 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -210,6 +210,9 @@ ## ๋ชจ๋ธ ๋ ˆ์ง€์ŠคํŠธ๋ฆฌ์™€ ํŒฉํ† ๋ฆฌ +- **2026-11-01** ํ•„ํ„ฐยท์ธ์‚ฌ๋งยท์ œ๋ชฉ ์ •๋ฆฌ ํ˜ธ์ถœ์— ์ถœ๋ ฅ ์ƒํ•œ(ํ•„ํ„ฐ 8,192ํ† ํฐ)์„ ๋’€์Šต๋‹ˆ๋‹ค. ๊ธฐ๋ณธ ์ถ”๋ก  + ๊นŠ์ด์—์„œ ํ•„ํ„ฐ ํ˜ธ์ถœ 42๊ฑด์˜ ์ถœ๋ ฅ์€ ์ตœ๋Œ€ 214ํ† ํฐ(์ค‘์•™๊ฐ’ 168)์ด์—ˆ๊ณ , ๊ฐ€์žฅ ๊ธด prod ํ›„๋ณด 5๊ฑด์„ + `thinking_effort: max`๋กœ ์ฑ„์ ํ•ด๋„ ์ตœ๋Œ€ 1,944ํ† ํฐ์ด๋ผ ์ƒํ•œ์— ๋‹ฟ์ง€ ์•Š์•˜๊ณ  ์‹คํŒจ๋„ ์—†์—ˆ์Šต๋‹ˆ๋‹ค. - ๋ชจ๋ธ ํŒฉํ† ๋ฆฌ๊ฐ€ `utils.py`์— ์žˆ์–ด ๊ทธ ํŒŒ์ผ์ด 838์ค„๋กœ ๊ฐ€์žฅ ํฐ ๋ชจ๋“ˆ์ด์—ˆ์Šต๋‹ˆ๋‹ค. ํŒฉํ† ๋ฆฌ(์•ฝ 520์ค„)๋ฅผ `model_factory.py`๋กœ ๋ถ„๋ฆฌํ–ˆ์Šต๋‹ˆ๋‹ค. - "๋ ˆ๊ฑฐ์‹œ ๋ชจ๋ธ์€ ๋ ˆ์ง€์ŠคํŠธ๋ฆฌ ํ•ญ๋ชฉ์ด ์—†์–ด๋„ ๋œ๋‹ค"๋Š” ํ…Œ์ŠคํŠธ ์˜ˆ์™ธ ์•„๋ž˜์— ํ•ญ๋ชฉ์ด ์•„์˜ˆ ์—†๋Š” enum @@ -472,8 +475,14 @@ ์ˆ˜์น˜๋Š” ๐Ÿ› ๏ธ์— ๋‘˜ ์ˆ˜ ์žˆ๊ฒŒ ํ–ˆ์Šต๋‹ˆ๋‹ค. ์ฝ”๋“œ๊ฐ€ ์ด๋ฏธ ๊ณ ์น˜๋Š” ์ข…๊ฒฐ ์ฝœ๋ก  ๊ทœ์น™๊ณผ, ๋‹ค์Œ ์ค„๊ณผ ๋ชจ์ˆœ๋˜๋˜ "๊ธฐ์ˆ  ์šฉ์–ด๋ฅผ ๊ฐ€๋Šฅํ•˜๋ฉด ๋ฒˆ์—ญํ•˜๋ผ"๋Š” ์ค„์„ ์ง€์› ์Šต๋‹ˆ๋‹ค. ๊ฒ€์ถœ๊ธฐ๋Š” `CLICHE_MARKERS`์— ํ‘œ์ง€ ๋‘˜, ๋น ์ง„ ๊ฒƒ์— ๋Œ€ํ•œ ๋ฌธ์žฅ ํŒ์ •์— ์–ด๋ฏธ ๋‘˜์„ ๋”ํ–ˆ์Šต๋‹ˆ๋‹ค. 2026-10-10 ์ €์žฅ ์‹คํ–‰์„ ๋‘ ๋ฒˆ ๋‹ค์‹œ ์ƒ์„ฑํ•˜๋‹ˆ overall์ด - 0.79์—์„œ ๋‘ ๋ฒˆ ๋ชจ๋‘ 0.95๊ฐ€ ๋๊ณ , length๋Š” 0.99์—์„œ 0.80ยท0.91๋กœ ๋‚ด๋ ค๊ฐ”์Šต๋‹ˆ๋‹ค. ๋‘ ๋ฒˆ ๋ชจ๋‘ ํ•œ ํŽธ์ด - 6,000์ž ์˜ˆ์‚ฐ์„ ๋„˜์—ˆ์Šต๋‹ˆ๋‹ค. + 0.79์—์„œ 0.95๊ฐ€ ๋์ง€๋งŒ, ๊ฐ™์€ ๋‚  ๋ฐ”๊พธ๊ธฐ ์ „ ํ”„๋กฌํ”„ํŠธ๋กœ ๋‘ ๋ฒˆ ์ƒ์„ฑํ•ด๋„ 0.95์˜€์Šต๋‹ˆ๋‹ค. ์ด ์ƒ์Šน์€ + ํ”„๋กฌํ”„ํŠธ ๋ณ€๊ฒฝ์ด ์•„๋‹ˆ๋ผ ์ €์žฅ๋œ ๊ธฐ์ค€ ์‹คํ–‰ ์ดํ›„์˜ ๋ชจ๋ธ ์ถœ๋ ฅ ๋ณ€ํ™”์ด๊ณ , ์ด ๋ณ€๊ฒฝ์˜ ํšจ๊ณผ๋Š” ์ด ์ธก์ •์œผ๋กœ + ํ™•์ธ๋˜์ง€ ์•Š์•˜์Šต๋‹ˆ๋‹ค. ํ•œ ํŽธ(Pinterest, ์•ฝ 7,500์ž)์€ ๋‘ ํ”„๋กฌํ”„ํŠธ ๋ชจ๋‘์—์„œ 6,000์ž ์˜ˆ์‚ฐ์„ + ๋„˜์—ˆ์Šต๋‹ˆ๋‹ค. +- **2026-11-01** ํ”„๋กฌํ”„ํŠธ๋งŒ์œผ๋กœ๋Š” ๊ทธ ๊ธ€์ด ์˜ˆ์‚ฐ์„ ๋„˜์—ˆ์œผ๋ฏ€๋กœ, ์ƒํ•œ์˜ 1.1๋ฐฐ๋ฅผ ๋„˜๋Š” ์š”์•ฝ์€ ์ธก์ •๊ฐ’์„ + ๋„ฃ์–ด ํ•œ ๋ฒˆ ์ค„์ด๊ฒŒ ํ–ˆ์Šต๋‹ˆ๋‹ค(`SummaryLengthPrompt`). ๊ฐ™์€ ์‹คํ–‰์„ ๋‹ค์‹œ ์ƒ์„ฑํ•˜์ž 7,682์ž๊ฐ€ 6,041์ž๊ฐ€ + ๋๊ณ  specificity๋Š” 0.55 ๊ทธ๋Œ€๋กœ์˜€์Šต๋‹ˆ๋‹ค. ์ธ์‚ฌ๋ง ์ˆ˜์ • ํ”„๋กฌํ”„ํŠธ์— ๋‚จ์•„ ์žˆ๋˜ "์•„ํ‹ฐํด๋กœ ๋„˜๊ธฐ๋Š” ๋งˆ์ง€๋ง‰ + ๋ฌธ์žฅ ์œ ์ง€"๋„ ์ง€์› ์Šต๋‹ˆ๋‹ค(๊ทธ ์š”๊ตฌ๋Š” ์ธ์‚ฌ๋ง ํ”„๋กฌํ”„ํŠธ์—์„œ ์ด๋ฏธ ๋น ์กŒ์Šต๋‹ˆ๋‹ค). ## ์ถœ๋ ฅ ํ’ˆ์งˆ ๋ฃจ๋ธŒ๋ฆญ (`quality_metrics.py`) @@ -642,6 +651,8 @@ Outlook์ด ์ธ๋ผ์ธ ์š”์†Œ์˜ ํŒจ๋”ฉ์„ ๋ฌด์‹œํ•ด ๊ฐ€๋Š” ๊ฐ•์กฐ ์ค„๋กœ ๊ทธ๋ ค์„œ, ์ƒ‰๊ณผ ํŒจ๋”ฉ์„ ํ‘œ ์…€๋กœ ์˜ฎ๊ฒผ์Šต๋‹ˆ๋‹ค. Google Fonts ์š”์ฒญ์„ ์ง€์šฐ๊ณ  ์ฝ”๋“œ ๊ธ€๊ผด์€ ์‹œ์Šคํ…œ ๊ณ ์ •ํญ์„ ์”๋‹ˆ๋‹ค. ๋ฉ”์ผ์— HTML๊ณผ ํ•จ๊ป˜ `text/plain` ๋Œ€์•ˆ ํŒŒํŠธ๋ฅผ ์‹ค์–ด ๋ณด๋ƒ…๋‹ˆ๋‹ค. +- **2026-11-01** ๋ชจ๋“  ์š”์•ฝ ์ด๋ฏธ์ง€๋ฅผ 712px๋กœ ๋งž์ถ”์ž 300px ๊ทธ๋ฆผ๋„ ๋Š˜์–ด๋‚ฌ์Šต๋‹ˆ๋‹ค. ์ด๋ฏธ์ง€ ํ™•์ธ์ด ์•ž๋ถ€๋ถ„ + 64KB์—์„œ ์‹ค์ œ ๋„ˆ๋น„๋ฅผ ์ฝ๊ณ , ์—ด๋ณด๋‹ค ์ข์€ ์ด๋ฏธ์ง€๋Š” ์ž๊ธฐ ๋„ˆ๋น„๋กœ ๋‘ก๋‹ˆ๋‹ค. ## ์˜ค์ผ€์ŠคํŠธ๋ ˆ์ด์…˜, ์•Œ๋ฆผ, ์ „๋‹ฌ diff --git a/docs/design.md b/docs/design.md index 2e827e6..635ee36 100644 --- a/docs/design.md +++ b/docs/design.md @@ -999,6 +999,13 @@ KO์˜ ์›์น™ 6ยท7์€ ๋ฌธ์ฒด ๊ทœ์น™์ด๊ณ  ์šฐ์„ ์ˆœ์œ„๊ฐ€ ๋ถ„๋Ÿ‰ยท๊ตฌ์ฒด์„ฑ(์› ๋ช…์‹œํ•ฉ๋‹ˆ๋‹ค. ๋ถ„๋Ÿ‰์„ ๋งž์ถ”๋ ค๊ณ  ๋ฉ”์ปค๋‹ˆ์ฆ˜ยทํŒŒ๋ผ๋ฏธํ„ฐยท์ธก์ •๊ฐ’ยท์„ค๊ณ„ ํŠธ๋ ˆ์ด๋“œ์˜คํ”„๋ฅผ ํฌ์ƒํ•˜์ง€ ๋ง๊ณ , ์ค„์ผ ๋•Œ๋Š” ์ค‘๋ณต ์„œ์ˆ ์„ ์ž๋ฅด๊ฒŒ ํ•ฉ๋‹ˆ๋‹ค. +ํ”„๋กฌํ”„ํŠธ๋งŒ์œผ๋กœ๋Š” ์ƒํ•œ์„ ์ง€ํ‚ค์ง€ ๋ชปํ•˜๋Š” ๊ธ€์ด ์žˆ์–ด ์ฝ”๋“œ๊ฐ€ ๊ธธ์ด๋ฅผ ์žฝ๋‹ˆ๋‹ค. ์™„์„ฑ๋œ ์š”์•ฝ์ด ์ƒํ•œ(KO +6,000์ž, EN 2,000๋‹จ์–ด)์˜ 1.1๋ฐฐ๋ฅผ ๋„˜์œผ๋ฉด ์ธก์ •๊ฐ’์„ ๋„ฃ์–ด ๊ฐ™์€ ์š”์•ฝ ๋ชจ๋ธ์— ํ•œ ๋ฒˆ ์ค„์—ฌ ๋‹ฌ๋ผ๊ณ  +ํ•ฉ๋‹ˆ๋‹ค(`SummaryLengthPrompt`, `Summarizer._fit_length`). ์ธก์ •๊ฐ’ยทํŒŒ๋ผ๋ฏธํ„ฐยท์ฝ”๋“œยทํ‘œ์™€ ์„น์…˜ ๊ตฌ์„ฑ์€ +์ง€์šฐ์ง€ ๋ง๋ผ๊ณ  ์ง€์‹œํ•˜๊ณ , ๊ฒฐ๊ณผ๋Š” ๊ฐ™์€ ์ •๋ฆฌ ๊ณผ์ •์„ ๋‹ค์‹œ ๊ฑฐ์นœ ๋’ค ๋” ์งง๊ณ  ์“ธ ์ˆ˜ ์žˆ์„ ๋•Œ๋งŒ ๋ฐ›์Šต๋‹ˆ๋‹ค. +์ค„์ด๊ธฐ๊ฐ€ ์‹คํŒจํ•˜๊ฑฐ๋‚˜ ๋” ๋‚˜์˜๋ฉด ์›๋ž˜ ์š”์•ฝ์„ ๊ทธ๋Œ€๋กœ ์”๋‹ˆ๋‹ค. ๊ธด ์š”์•ฝ์€ ํ ์ด์ง€๋งŒ ์žƒ์€ ์š”์•ฝ์€ ์†์‹ค์ด๊ธฐ +๋•Œ๋ฌธ์ž…๋‹ˆ๋‹ค. + ๐Ÿ“Š ์™ธ์˜ ์„น์…˜์€ ์„ฑ๊ณผ๋ฅผ ๋ง๋กœ ์„œ์ˆ ํ•˜๋˜ "์•„๋ž˜ ๐Ÿ“Š์— ์ •๋ฆฌํ–ˆ์Šต๋‹ˆ๋‹ค"์ฒ˜๋Ÿผ ๋‹ค๋ฅธ ์„น์…˜์„ ๊ฐ€๋ฆฌํ‚ค์ง€ ์•Š๊ฒŒ ํ•ฉ๋‹ˆ๋‹ค. ์†Œ์ œ๋ชฉ(h4-h6)์€ ๊ธˆ์ง€ํ•˜๋˜ ๋‹จ๋ฝ ์ฒซ๋จธ๋ฆฌ์˜ ๊ตต์€ ํ‘œ์ง€๋Š” ํ—ˆ์šฉํ•˜๊ณ , ์ธ๋ผ์ธ ์ฝ”๋“œ๋Š” ``๋กœ, ์ˆ˜์‹์€ LaTeX๊ฐ€ ์•„๋‹ˆ๋ผ ์ผ๋ฐ˜ ํ…์ŠคํŠธ๋‚˜ ์œ ๋‹ˆ์ฝ”๋“œ๋กœ ์“ฐ๊ฒŒ ํ•ฉ๋‹ˆ๋‹ค. @@ -1131,11 +1138,12 @@ lazy-loading์ฒ˜๋Ÿผ ํŒŒ์„œ๊ฐ€ ๋†“์นœ ์ด๋ฏธ์ง€๋„ ๋‘ ๋ฒˆ์งธ ์กฐ๊ฑด์œผ๋กœ ์‚ด์•„ src(๋ชจ๋ธ์ด macOS ์Šคํฌ๋ฆฐ์ƒท ์ด๋ฆ„์˜ U+202F๋ฅผ ์ผ๋ฐ˜ ๊ณต๋ฐฑ์œผ๋กœ ์˜ฎ๊ธด ๊ฒฝ์šฐ)๋Š” ์›๋ฌธ URL๋กœ ๋˜๋Œ๋ฆฝ๋‹ˆ๋‹ค. ์ถœ์ฒ˜๊ฐ€ ๋งž์•„๋„ ๋ฉ”์ผ์—์„œ ๋ณด์ด์ง€ ์•Š๋Š” ์ด๋ฏธ์ง€๊ฐ€ ์žˆ์Šต๋‹ˆ๋‹ค. `_drop_unrenderable_images`๋Š” ๋‚จ์€ -์ด๋ฏธ์ง€๋ฅผ ํ•˜๋‚˜์”ฉ ๋ฐ›์•„ ๋ณด๊ณ (`http.is_email_renderable_image`, ์š”์•ฝ๊ธฐ๋Š” ์ƒ์„ฑ์ž ์ธ์ž `image_probe`๋กœ ๋ฐ›์Šต๋‹ˆ๋‹ค), ์‘๋‹ต์ด PNGยทJPEGยทGIF๊ฐ€ ์•„๋‹ˆ๊ฑฐ๋‚˜ +์ด๋ฏธ์ง€๋ฅผ ํ•˜๋‚˜์”ฉ ๋ฐ›์•„ ๋ณด๊ณ (`http.probe_email_image`, ์š”์•ฝ๊ธฐ๋Š” ์ƒ์„ฑ์ž ์ธ์ž `image_probe`๋กœ ๋ฐ›์Šต๋‹ˆ๋‹ค), ์‘๋‹ต์ด PNGยทJPEGยทGIF๊ฐ€ ์•„๋‹ˆ๊ฑฐ๋‚˜ ๋ฐ›์•„์ง€์ง€ ์•Š์œผ๋ฉด ์›๋ฌธ์ด ๊ฐ™์€ ๊ทธ๋ฆผ์— ๋Œ€ํ•ด ์ ์–ด ๋‘” ๋‹ค๋ฅธ ํŒ๋ณธ(`srcset`, `data-src`, ``์˜ ``)์„ ์ฐจ๋ก€๋กœ ํ™•์ธํ•ด ๋ฉ”์ผ์—์„œ ๋ณด์ด๋Š” ์ฒซ ํŒ๋ณธ์œผ๋กœ ๋ฐ”๊ฟ‰๋‹ˆ๋‹ค. ๋ฉ”์ผ ํ™”๋ฉด์—๋Š” 2๋ฐฐ ํ•ด์ƒ๋„๋ฉด ์ถฉ๋ถ„ํ•˜๋ฏ€๋กœ 1600px ์ดํ•˜ ์ค‘ ๊ฐ€์žฅ ํฐ ๊ฒƒ๋ถ€ํ„ฐ ๋ด…๋‹ˆ๋‹ค. ๋งž๋Š” ํŒ๋ณธ์ด ์—†์œผ๋ฉด figure๋ฅผ ์บก์…˜์งธ ์ง€์›๋‹ˆ๋‹ค. ํ™•์ธ์€ ์ด๋ฏธ์ง€๋‹น -ํ•œ ๋ฒˆ, 10์ดˆ ์•ˆ์— ํ—ค๋”๋งŒ ๋ฐ›๊ณ (`IMAGE_PROBE_TIMEOUT_SECONDS`), ์š”์•ฝ ํ•˜๋‚˜๊ฐ€ ์“ฐ๋Š” ํ™•์ธ ์‹œ๊ฐ„์€ ๋ชจ๋‘ ํ•ฉ์ณ +ํ•œ ๋ฒˆ, 10์ดˆ ์•ˆ์— ์‘๋‹ต ํ—ค๋”์™€ ์ด๋ฏธ์ง€ ์•ž๋ถ€๋ถ„ 64KB๋งŒ ๋ฐ›์•„ ๋„ˆ๋น„๋ฅผ ์ฝ๊ณ (`IMAGE_PROBE_TIMEOUT_SECONDS`, +`IMAGE_HEADER_BYTES`), ์š”์•ฝ ํ•˜๋‚˜๊ฐ€ ์“ฐ๋Š” ํ™•์ธ ์‹œ๊ฐ„์€ ๋ชจ๋‘ ํ•ฉ์ณ 120์ดˆ(`IMAGE_PROBE_BUDGET_SECONDS`)์ž…๋‹ˆ๋‹ค. ๊ทธ ์‹œ๊ฐ„์„ ๋„˜๊ธฐ๋ฉด ์•„์ง ํ™•์ธํ•˜์ง€ ๋ชปํ•œ ์ด๋ฏธ์ง€๋Š” ๋ณด๋‚ด์ง€ ์•Š๊ณ  ์ง€์›๋‹ˆ๋‹ค. Windows Outlook์€ WebPยทAVIF ์ž๋ฆฌ์— alt ํ…์ŠคํŠธ๋ฅผ ๋ณด์—ฌ ์ฃผ๋ฏ€๋กœ, ๊ทธ๋Œ€๋กœ ๋‘๋ฉด alt ํ…์ŠคํŠธ์™€ ์บก์…˜์ด ํ•œ ๋ฌธ์žฅ์ฒ˜๋Ÿผ ์ด์–ด์ ธ ๋ณด์ž…๋‹ˆ๋‹ค. URL ํ™•์žฅ์ž๋กœ ์ถ”์ธกํ•˜์ง€ @@ -1181,7 +1189,9 @@ SSRF ๊ฒ€์‚ฌ๋ฅผ ๊ฑฐ์นฉ๋‹ˆ๋‹ค. ๋‚จ์€ src๋Š” `_encode_image_srcs`๊ฐ€ URL์— ๊ทธ๋Œ€ ํ…Œ๋‘๋ฆฌ์™€ ํŒจ๋”ฉ์€ ํ…œํ”Œ๋ฆฟ ๊ทœ์น™์ด ์ค๋‹ˆ๋‹ค(ยง14). ๋ชจ๋“  ``์—๋Š” `width="712"`์™€ `width:100%;max-width:712px;height:auto` ์ธ๋ผ์ธ ์Šคํƒ€์ผ์„ ๋ถ™์ž…๋‹ˆ๋‹ค(`SUMMARY_IMAGE_WIDTH`, 800px ์—ด์—์„œ ์นด๋“œ ์—ฌ๋ฐฑ๊ณผ ํŒจ๋”ฉ์„ ๋บ€ ๊ฐ’). Windows Outlook์€ `max-width`๋ฅผ ๋ฌด์‹œํ•˜๋ฏ€๋กœ `width` ์†์„ฑ์ด -์—†์œผ๋ฉด 2,000px ์ฐจํŠธ๊ฐ€ ์—ด ์ „์ฒด๋ฅผ ์ฐฝ ๋ฐ–์œผ๋กœ ๋ฐ€์–ด๋ƒ…๋‹ˆ๋‹ค. +์—†์œผ๋ฉด 2,000px ์ฐจํŠธ๊ฐ€ ์—ด ์ „์ฒด๋ฅผ ์ฐฝ ๋ฐ–์œผ๋กœ ๋ฐ€์–ด๋ƒ…๋‹ˆ๋‹ค. ์ด๋ฏธ์ง€ ํ™•์ธ ๋‹จ๊ณ„๊ฐ€ ์‹ค์ œ ๋„ˆ๋น„๋ฅผ ์ฝ์–ด +๊ทธ๋ณด๋‹ค ์ข์€ ์ด๋ฏธ์ง€์—๋Š” ์ž๊ธฐ ๋„ˆ๋น„์™€ `max-width:100%;height:auto`๋ฅผ ๋‹ค์‹œ ์ค๋‹ˆ๋‹ค. ๊ทธ๋ž˜์„œ ์ž‘์€ ๊ทธ๋ฆผ์ด +712px๋กœ ๋Š˜์–ด๋‚˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. **ํ‰๋ฌธ์œผ๋กœ ์ฝ๋Š” ํƒœ๊ทธ.** ํ•„ํ„ฐ ์‘๋‹ต์˜ `score`, `reason`๊ณผ ์š”์•ฝ์˜ `tags`๋Š” ํ‰๋ฌธ์œผ๋กœ ์ฝ์Šต๋‹ˆ๋‹ค(`text_tags`). ํƒœ๊ทธ ์•ˆ์˜ HTML์„ ์ง๋ ฌํ™”ํ•ด ๋Œ๋ ค์ฃผ๋ฉด "Q&A"๊ฐ€ "Q&A"๋กœ ์˜ค๊ณ , diff --git a/docs/diagrams/pipeline-flow.png b/docs/diagrams/pipeline-flow.png index 957a0b2..df698b9 100644 Binary files a/docs/diagrams/pipeline-flow.png and b/docs/diagrams/pipeline-flow.png differ diff --git a/scripts/eval_filtering.py b/scripts/eval_filtering.py index f098462..1604cb7 100644 --- a/scripts/eval_filtering.py +++ b/scripts/eval_filtering.py @@ -68,7 +68,7 @@ def _score_live( os.environ.setdefault("CONFIG_FILE_SUFFIX", stage) from app.adapters.bedrock import BedrockLanguageModelFactory - from app.adapters.http import is_email_renderable_image + from app.adapters.http import probe_email_image from app.application.summarize import Summarizer, SummarizerSettings from app.configs import Config from app.domain.constants import FilteringCriteria @@ -91,7 +91,7 @@ def _score_live( | {"filtering_criteria": FilteringCriteria(data.get("criteria", "all"))} | ({"filtering_model_id": model} if model else {}) ), - is_email_renderable_image, + probe_email_image, ) included = ", ".join(data.get("included_topics", [])) excluded = ", ".join(data.get("excluded_topics", [])) diff --git a/scripts/eval_summary_quality.py b/scripts/eval_summary_quality.py index d4eceb2..aed5b45 100644 --- a/scripts/eval_summary_quality.py +++ b/scripts/eval_summary_quality.py @@ -150,7 +150,7 @@ def regenerate(date: str, stage: str, profile: str | None, suffix: str) -> str: os.environ.setdefault("CONFIG_FILE_SUFFIX", stage) from app.adapters.bedrock import BedrockLanguageModelFactory - from app.adapters.http import is_email_renderable_image + from app.adapters.http import probe_email_image from app.application.summarize import Summarizer, SummarizerSettings from app.configs import Config from app.domain.post import Post @@ -174,7 +174,7 @@ def regenerate(date: str, stage: str, profile: str | None, suffix: str) -> str: summarizer = Summarizer( BedrockLanguageModelFactory(boto_session=session), SummarizerSettings.model_validate(config.summarization.model_dump()), - is_email_renderable_image, + probe_email_image, ) posts = [ Post( diff --git a/scripts/replay_filter.py b/scripts/replay_filter.py index b22912c..3e6fcd3 100644 --- a/scripts/replay_filter.py +++ b/scripts/replay_filter.py @@ -121,7 +121,7 @@ def score( import boto3 from app.adapters.bedrock import BedrockLanguageModelFactory - from app.adapters.http import is_email_renderable_image + from app.adapters.http import probe_email_image from app.application.summarize import Summarizer, SummarizerSettings from app.domain.post import Post from app.domain.summary import MODEL_FAILURE_PREFIXES @@ -151,7 +151,7 @@ def score( summarizer = Summarizer( BedrockLanguageModelFactory(boto_session=session), settings, - is_email_renderable_image, + probe_email_image, ) posts = [ Post( diff --git a/tests/test_pipeline_guards.py b/tests/test_pipeline_guards.py index dc3efba..d672841 100644 --- a/tests/test_pipeline_guards.py +++ b/tests/test_pipeline_guards.py @@ -451,7 +451,7 @@ class TestHandlerWiring: def test_the_summarizer_gets_the_real_image_probe(self, monkeypatch): # No fixture post carries an image, so a probe that dropped every image # would pass the golden snapshot; pin the wiring instead. - from app.adapters.http import is_email_renderable_image + from app.adapters.http import probe_email_image from app.domain.crawl import CrawlReport seen = {} @@ -474,7 +474,7 @@ class resources: monkeypatch.setattr(main, "log_run_summary", lambda *a: None) monkeypatch.setattr(main, "fetch_and_filter_posts", _fetch) assert main.handler({}, None)["statusCode"] == 200 - assert seen["probe"] is is_email_renderable_image + assert seen["probe"] is probe_email_image class TestModelFailureAlertFromTheHandler: diff --git a/tests/test_request_layer.py b/tests/test_request_layer.py index b0d6199..9c947f2 100644 --- a/tests/test_request_layer.py +++ b/tests/test_request_layer.py @@ -197,11 +197,11 @@ def fake_getaddrinfo(host, *a, **k): "url", ["http://169.254.169.254/x.png", "http://2130706433/x.png"] ) def test_an_image_on_a_blocked_host_is_never_requested(self, monkeypatch, url): - from app.adapters.http import is_email_renderable_image + from app.adapters.http import probe_email_image session = _FakeSession([_FakeResponse(200)]) _patch_session(monkeypatch, session) - assert not is_email_renderable_image(url) + assert probe_email_image(url) is None assert session.calls == 0 def test_allows_hostname_resolving_to_public_ip(self, monkeypatch): @@ -234,11 +234,17 @@ def respond(self, monkeypatch): calls: list[dict] = [] - def install(content_type="image/png", ok=True, raises=None): + def install(content_type="image/png", ok=True, raises=None, body=b""): class _Resp: def __init__(self): self.ok = ok self.headers = {"content-type": content_type} + self.read = 0 + + def iter_content(self, chunk_size): + for start in range(0, len(body), chunk_size): + self.read += 1 + yield body[start : start + chunk_size] def __enter__(self): return self @@ -261,26 +267,76 @@ def fake(session, url, headers, timeout=None, stream=False): "content_type", ["image/png", "image/jpeg; q=1", "image/gif"] ) def test_email_safe_types_are_kept(self, respond, content_type): - from app.adapters.http import is_email_renderable_image + from app.adapters.http import probe_email_image respond(content_type) - assert is_email_renderable_image("https://cdn.example.com/a") + # Kept, with an unknown width: the fake serves no image bytes. + assert probe_email_image("https://cdn.example.com/a") == 0 + + @pytest.mark.parametrize("fmt", ["PNG", "JPEG", "GIF"]) + def test_the_width_comes_from_the_image_header(self, respond, fmt): + import io + + from PIL import Image + + from app.adapters.http import probe_email_image + + buffer = io.BytesIO() + Image.new("RGB", (120, 40)).save(buffer, format=fmt) + respond(f"image/{fmt.lower()}", body=buffer.getvalue()) + assert probe_email_image("https://cdn.example.com/a") == 120 @pytest.mark.parametrize( "content_type", ["image/webp", "image/avif", "image/svg+xml", "application/octet-stream"], ) def test_types_mail_clients_do_not_show_are_dropped(self, respond, content_type): - from app.adapters.http import is_email_renderable_image + from app.adapters.http import probe_email_image respond(content_type) - assert not is_email_renderable_image("https://cdn.example.com/a") + assert probe_email_image("https://cdn.example.com/a") is None def test_an_image_that_does_not_load_is_dropped(self, respond): - from app.adapters.http import is_email_renderable_image + from app.adapters.http import probe_email_image respond(ok=False) - assert not is_email_renderable_image("https://cdn.example.com/a") + assert probe_email_image("https://cdn.example.com/a") is None + + def test_only_the_header_bytes_are_read(self, respond): + from app.adapters import http + + respond(body=b"\x00" * (10 * http.IMAGE_HEADER_BYTES)) + response_reads = [] + original = http._image_width + + def spy(response, deadline): + width = original(response, deadline) + response_reads.append(response.read) + return width + + http._image_width = spy + try: + assert http.probe_email_image("https://cdn.example.com/a") == 0 + finally: + http._image_width = original + assert response_reads == [http.IMAGE_HEADER_BYTES // (16 * 1024)] + + def test_a_header_claiming_a_huge_image_is_kept_with_unknown_width(self, respond): + # Pillow refuses a header past its decompression-bomb limit; that must + # not drop the image, let alone the card around it. + import struct + import zlib + + from app.adapters.http import probe_email_image + + def chunk(kind: bytes, data: bytes) -> bytes: + crc = struct.pack(">I", zlib.crc32(kind + data)) + return struct.pack(">I", len(data)) + kind + data + crc + + ihdr = struct.pack(">IIBBBBB", 20000, 10000, 8, 2, 0, 0, 0) + png = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", b"") + respond(body=png) + assert probe_email_image("https://cdn.example.com/a") == 0 def test_one_short_header_only_attempt(self, respond): # A hung host used to cost 3 attempts x 30 s per image, with the body @@ -289,14 +345,14 @@ def test_one_short_header_only_attempt(self, respond): from app.adapters.http import ( IMAGE_PROBE_TIMEOUT_SECONDS, - is_email_renderable_image, + probe_email_image, ) calls = respond(raises=requests.exceptions.ConnectTimeout("hung")) - assert not is_email_renderable_image("https://cdn.example.com/a") + assert probe_email_image("https://cdn.example.com/a") is None assert calls == [{"timeout": IMAGE_PROBE_TIMEOUT_SECONDS, "stream": True}] def test_an_unsafe_scheme_is_never_fetched(self): - from app.adapters.http import is_email_renderable_image + from app.adapters.http import probe_email_image - assert not is_email_renderable_image("javascript:alert(1)") + assert probe_email_image("javascript:alert(1)") is None diff --git a/tests/test_summarizer_internals.py b/tests/test_summarizer_internals.py index 1f94457..d52f4e5 100644 --- a/tests/test_summarizer_internals.py +++ b/tests/test_summarizer_internals.py @@ -58,7 +58,7 @@ def _summarizer(min_score=0.7, **overrides): s.language = s.settings.language s.filtered_out_posts = [] # Every image loads; tests about the mail-client check pass their own probe. - s.image_probe = lambda url: True + s.image_probe = lambda url: 0 from app.application.llm_batching import BatchProcessor s.batch_processor = BatchProcessor(max_concurrency=2, batch_size=10) @@ -339,7 +339,7 @@ def test_an_image_mail_clients_cannot_show_is_dropped_with_its_caption( webp = "https://cdn.example.com/chart.width-100.format-webp.webp" png = "https://cdn.example.com/arch.png" s = _summarizer() - s.image_probe = lambda url: url == png + s.image_probe = lambda url: 0 if url == png else None post = _post("a", content=f'') post.images = [webp, png] s.summarizer = _FakeChain( @@ -365,7 +365,7 @@ def test_a_webp_is_swapped_for_a_png_rendition_the_article_lists(self, monkeypat f'{base}-1536.png 1536w, {base}-2048.png 2048w">' ) s = _summarizer() - s.image_probe = lambda url: url.endswith(".png") + s.image_probe = lambda url: 0 if url.endswith(".png") else None post = _post("a", content=source) post.images = [f"{base}.webp"] s.summarizer = _FakeChain( @@ -391,12 +391,35 @@ def test_images_not_checked_within_the_budget_are_dropped(self, monkeypatch): html, dropped = module._drop_unrenderable_images( '', "", - is_renderable=lambda url: True, + is_renderable=lambda url: 0, budget_seconds=60, clock=lambda: next(ticks), ) assert "a.png" in html and dropped == ["https://x/b.png"] + @pytest.mark.parametrize( + ("width", "attr", "style"), + [ + (120, "120", "max-width:100%;height:auto"), + (2369, "712", "width:100%;max-width:712px;height:auto"), + (0, "712", "width:100%;max-width:712px;height:auto"), + ], + ) + def test_a_narrow_image_keeps_its_own_width(self, width, attr, style): + from bs4 import BeautifulSoup + + from app.application import summarize as module + from app.domain.summary import SummaryOutput + + sanitized = SummaryOutput.model_validate( + {"summary": '

x

', "tags": [], "urls": []} + ).summary + html, _ = module._drop_unrenderable_images( + sanitized, "", is_renderable=lambda url: width + ) + img = BeautifulSoup(html, "html.parser").find("img") + assert (img["width"], img["style"]) == (attr, style) + def test_an_already_encoded_image_url_is_left_alone(self): url = "https://cdn.example.com/a%20b.png?w=1200&q=90" s = _summarizer() @@ -964,3 +987,56 @@ def test_escaped_text_in_a_span_stays_escaped(self): out = _postprocess_summary("

Check `a < b` here.

") assert out == "

Check a < b here.

" + + +class TestFitLength: + """An over-long summary is sent back once; nothing about it may lose the card.""" + + LONG = ( + "

" + "์ธก์ •๊ฐ’๊ณผ ๋ฉ”์ปค๋‹ˆ์ฆ˜์„ ์„ค๋ช…ํ•˜๋Š” ๊ธด ๋ฌธ์žฅ์ž…๋‹ˆ๋‹ค. " * 330 + "

" + ) # ~7,600 + + class _Shortener: + def __init__(self, answer): + self.answer, self.calls = answer, [] + + def invoke(self, inputs): + self.calls.append(inputs) + if isinstance(self.answer, Exception): + raise self.answer + return {"summary": self.answer} + + def _fit(self, monkeypatch, summary, answer): + from app.application import llm_batching + from app.domain.summary import SummaryOutput + + monkeypatch.setattr(llm_batching.time, "sleep", lambda _s: None) + s = _summarizer() + s.shortener = self._Shortener(answer) + output = SummaryOutput.model_validate({"summary": summary}) + finalized = s._finalize_summary(_post("a"), output.summary) + return s, s._fit_length(_post("a"), output, finalized) + + def test_a_summary_within_tolerance_is_not_sent_back(self, monkeypatch): + s, out = self._fit(monkeypatch, BODY, "unused") + assert s.shortener.calls == [] and out.startswith("

") + + def test_an_over_long_summary_is_shortened_once(self, monkeypatch): + short = "

" + "์ค„์ธ ์š”์•ฝ ๋ฌธ์žฅ์ž…๋‹ˆ๋‹ค. " * 300 + "

" + s, out = self._fit(monkeypatch, self.LONG, short) + assert len(s.shortener.calls) == 1 + assert s.shortener.calls[0]["limit"] == 6000 + assert s.shortener.calls[0]["measured"] > 6600 + assert "์ค„์ธ ์š”์•ฝ" in out + + @pytest.mark.parametrize( + "answer", + [ + ValueError("malformed"), + "

" + "๋” ๊ธธ๊ฒŒ ๋Š˜๋ฆฐ ๋ฌธ์žฅ์ž…๋‹ˆ๋‹ค. " * 600 + "

", + "

์งง์Œ

", # below the usable floor + ], + ) + def test_a_failed_or_worse_rewrite_keeps_the_original(self, monkeypatch, answer): + _, out = self._fit(monkeypatch, self.LONG, answer) + assert "์ธก์ •๊ฐ’๊ณผ ๋ฉ”์ปค๋‹ˆ์ฆ˜" in out