fix: residuals — image widths, summary length, diagram export

app/adapters/http.py

@@ -4,13 +4,16 @@
 redirect hop before connecting.
 """
 
+import io
 import ipaddress
 import socket
 import time
+import warnings
 from urllib.parse import urljoin, urlparse
 
 import requests
 from bs4 import BeautifulSoup
+from PIL import Image, UnidentifiedImageError
 from requests.exceptions import RequestException
 
 from app.domain.post import is_safe_url
@@ -242,19 +245,23 @@ def _make_robust_request(url: str) -> requests.Response | None:
 # One short attempt per image: a static CDN either answers at once or the image
 # is not worth holding the run for (the summarizer also caps the total).
 IMAGE_PROBE_TIMEOUT_SECONDS = 10
+# Enough of the file for PNG, GIF and JPEG headers (a JPEG's size marker can sit
+# behind tens of KB of EXIF data); the rest of the image is never downloaded.
+IMAGE_HEADER_BYTES = 64 * 1024
 
 
-def is_email_renderable_image(url: str) -> bool:
-    """Whether ``url`` serves an image a mail client will display.
+def probe_email_image(url: str) -> int | None:
+    """The pixel width of the image at ``url`` if a mail client will display
+    it (0 when the width cannot be read), else None.
 
     Fetched, not guessed from the URL: CDN URLs often carry no extension, and
     the only proof an image loads is that it does. The request asks for the
     email-safe types, so a content-negotiating CDN answers with one of them if
-    it can. Only the headers are read. SSRF-checked at every redirect hop like
-    every other fetch.
+    it can. Only the first bytes are read. SSRF-checked at every redirect hop
+    like every other fetch.
     """
     if not is_safe_url(url):
-        return False
+        return None
     headers = REQUEST_HEADERS | {
         "Accept": "image/png,image/jpeg,image/gif;q=0.9,*/*;q=0.1"
     }
@@ -268,11 +275,42 @@ def is_email_renderable_image(url: str) -> bool:
         )
     except RequestException as e:
         logger.info("Image probe failed for '%s': %s", url, e)
-        return False
+        return None
     if response is None:
-        return False
+        return None
     with response:
         if not response.ok:
-            return False
+            return None
         content_type = response.headers.get("content-type", "").split(";")[0]
-        return content_type.strip().lower() in EMAIL_IMAGE_TYPES
+        if content_type.strip().lower() not in EMAIL_IMAGE_TYPES:
+            return None
+        return _image_width(response, time.monotonic() + IMAGE_PROBE_TIMEOUT_SECONDS)
+
+
+def _image_width(response: requests.Response, deadline: float) -> int:
+    """The width from the image's header bytes, or 0 if they do not say.
+
+    Only the header is parsed (``.size``), never the pixels. A header claiming a
+    huge image trips Pillow's decompression-bomb guard, which is still a
+    displayable image of unknown width, not a reason to lose the card.
+    """
+    head = b""
+    try:
+        for chunk in response.iter_content(chunk_size=16 * 1024):
+            head += chunk
+            # The timeout bounds each read, not the total: a host that drips
+            # bytes would otherwise hold the probe indefinitely.
+            if len(head) >= IMAGE_HEADER_BYTES or time.monotonic() >= deadline:
+                break
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", Image.DecompressionBombWarning)
+            with Image.open(io.BytesIO(head)) as image:
+                return int(image.size[0])
+    except (
+        RequestException,
+        UnidentifiedImageError,
+        Image.DecompressionBombError,
+        OSError,
+        ValueError,
+    ):
+        return 0

app/application/ports.py

@@ -113,5 +113,6 @@ def is_converse(self, model: Any) -> bool:
         differs from the native request body's."""
 
 
-#: Whether an image URL serves a type every mail client displays.
-ImageProbe = Callable[[str], bool]
+#: The pixel width of a mail-displayable image at a URL (0 if unknown), or None
+#: if mail clients would not display it.
+ImageProbe = Callable[[str], int | None]

app/application/prompts/__init__.py

@@ -4,6 +4,7 @@
     GreetingPrompt,
     GreetingRevisionPrompt,
     SummarizationPrompt,
+    SummaryLengthPrompt,
     TitleCasePrompt,
 )
 
@@ -13,5 +14,6 @@
     "GreetingPrompt",
     "GreetingRevisionPrompt",
     "SummarizationPrompt",
+    "SummaryLengthPrompt",
     "TitleCasePrompt",
 ]

app/application/prompts/prompts.py

@@ -428,8 +428,8 @@ class GreetingRevisionPrompt(BasePrompt):
 <draft>{draft}</draft>
 
 Count the words rather than estimating. If it is too long, start by dropping a whole sentence
-(three sentences including the opener are enough). Keep the opener "Hey friends! I'm Peccy 😎",
-the specific observation and the hand-off to the articles, and keep the casual confident tone.
+(three sentences including the opener are enough). Keep the opener "Hey friends! I'm Peccy 😎"
+and the specific observation, and keep the casual confident tone.
 Do NOT introduce a "back then X, now Y" contrast. Plain text only.
 
 **OUTPUT:** The revised introduction only.""",
@@ -440,8 +440,8 @@ class GreetingRevisionPrompt(BasePrompt):
 <draft>{draft}</draft>
 
 글자 수는 어림하지 말고 세세요. 너무 길면 문장 하나를 통째로 빼는 것부터 하세요(여는 인사
-포함 3문장이면 충분합니다). 여는 인사 "안녕 친구들! 난 Peccy야 😎"와 구체적인 관찰, 아티클로
-넘기는 마지막 문장은 유지하고, 반말·자신감 있는 어조도 그대로 두세요. "예전엔 X였는데 이제
+포함 3문장이면 충분합니다). 여는 인사 "안녕 친구들! 난 Peccy야 😎"와 구체적인 관찰은 유지하고,
+반말·자신감 있는 어조도 그대로 두세요. "예전엔 X였는데 이제
 Y야" 같은 과거-현재 대조는 넣지 마세요. 평문만 출력하세요.
 
 **OUTPUT:** 수정된 인트로만 출력.""",
@@ -883,3 +883,54 @@ def for_language(
             },
         )
         return prompt_class
+
+
+class SummaryLengthPrompt(BasePrompt):
+    """Ask the summarization model to bring an over-long summary under its budget.
+
+    The budget is stated in the summarization prompt, and the model still runs a
+    long, dense article a quarter over it. Length is something code measures
+    exactly, so an over-long summary is sent back once with the measured count.
+    """
+
+    input_variables: list[str] = ["summary", "measured", "limit"]
+    output_variables: list[str] = ["summary"]
+    system_prompt_template: str = SummarizationPrompt.system_prompt_template
+    human_prompt_template: str = ""
+
+    _human_prompt_template: ClassVar[dict[Language, str]] = {
+        Language.EN: """Your summary below is {measured} words of visible text. The limit is
+{limit} words, so shorten it to fit.
+
+<draft>{summary}</draft>
+
+Cut sentences that restate something already said, background the reader does not need, and
+the least important details first. Keep every measured number, parameter name, code block and
+table, the sections and their order, and the HTML markup and writing style the draft
+already follows. Add nothing new.
+
+**OUTPUT:** <summary>[the shortened summary]</summary> only.""",
+        Language.KO: """아래 요약은 가시 텍스트 기준 {measured}자입니다. 상한이 {limit}자이니 그 안으로
+줄이세요.
+
+<draft>{summary}</draft>
+
+앞에서 한 말을 되풀이한 문장, 독자에게 필요 없는 배경, 덜 중요한 세부부터 지우세요. 측정값,
+파라미터 이름, 코드 블록, 표는 지우지 말고 섹션 구성과 순서, HTML 마크업과 초안의 문체(합니다체,
+번역투·상투구 없이)도 그대로 두세요. 새 내용은 더하지 마세요.
+
+**OUTPUT:** <summary>[줄인 요약]</summary>만 출력.""",
+    }
+
+    @classmethod
+    def for_language(
+        cls, language: Language = Language.KO
+    ) -> type["SummaryLengthPrompt"]:
+        return type(
+            f"{language.name.capitalize()}SummaryLengthPrompt",
+            (cls,),
+            {
+                "system_prompt_template": cls.system_prompt_template,
+                "human_prompt_template": cls._human_prompt_template[language],
+            },
+        )

app/application/summarize.py

@@ -1,3 +1,4 @@
+import functools
 import time
 from collections.abc import Callable
 from typing import Any, ClassVar
@@ -11,9 +12,14 @@
     BatchProcessor,
     HTMLTagOutputParser,
     measure_execution_time,
+    retry_with_backoff,
 )
 from app.application.ports import ChatModelProvider, ImageProbe
-from app.application.prompts.prompts import FilteringPrompt, SummarizationPrompt
+from app.application.prompts.prompts import (
+    FilteringPrompt,
+    SummarizationPrompt,
+    SummaryLengthPrompt,
+)
 from app.domain.constants import Language, LanguageModelId
 from app.domain.content import content_html, visible_text
 from app.domain.eval_metrics import is_anchor
@@ -25,6 +31,7 @@
     SUMMARY_FAILED,
     SUMMARY_INVALID,
     SummarizationOptions,
+    SummarizerConfig,
     SummaryOutput,
     _encode_image_srcs,
     _image_candidates,
@@ -48,14 +55,15 @@ def _drop_unrenderable_images(
     budget_seconds: float = IMAGE_PROBE_BUDGET_SECONDS,
     clock: Callable[[], float] = time.monotonic,
 ) -> tuple[str, list[str]]:
-    """Keep only images a mail client will display.
+    """Keep only images a mail client will display, sized to their own width.
 
     Outlook for Windows shows a WebP image's alt text instead, so the alt text
     and the caption ran together as one odd sentence ("Massive Text Embedding
     Benchmark (Code)코드 검색 부문 ... 차트입니다"). An image that fails is
     swapped for another rendition the article lists (NVIDIA serves WebP as the
-    main file and PNG in its srcset); with none, its figure goes. Returns
-    ``(cleaned_html, dropped_srcs)``.
+    main file and PNG in its srcset); with none, its figure goes. A kept image
+    narrower than the column keeps its own width instead of being stretched to
+    it. Returns ``(cleaned_html, dropped_srcs)``.
     """
     if "<img" not in summary_html.lower():
         return summary_html, []
@@ -65,19 +73,24 @@ def _drop_unrenderable_images(
     changed = False
     for img in list(soup.find_all("img")):
         src = str(img.get("src") or "")
-        chosen = None
+        chosen, width = None, 0
         for candidate in [src, *_image_candidates(src, source_html)]:
             if clock() >= deadline:
                 break
-            if is_renderable(candidate):
-                chosen = candidate
+            if (probed := is_renderable(candidate)) is not None:
+                chosen, width = candidate, probed
                 break
         if chosen is None:
             dropped.append(src)
             (img.find_parent("figure") or img).decompose()
-        elif chosen != src:
+            continue
+        if chosen != src:
             img["src"] = chosen
             changed = True
+        if 0 < width < SummarizerConfig.SUMMARY_IMAGE_WIDTH:
+            img["width"] = str(width)
+            img["style"] = "max-width:100%;height:auto"
+            changed = True
     return (str(soup), dropped) if (dropped or changed) else (summary_html, [])
 
 
@@ -104,7 +117,8 @@ def __init__(
         image_probe: ImageProbe,
     ) -> None:
         self.settings = settings
-        self.image_probe = image_probe
+        # Cached per run: a shortened rewrite probes the same images again.
+        self.image_probe = functools.cache(image_probe)
         self.language = settings.language
         self.filtered_out_posts: list[tuple[Post, str]] = []
         self.batch_processor = BatchProcessor(
@@ -251,6 +265,11 @@ def _create_summarizer(
             and self._cache_can_hit(),
             use_converse=self.llm_factory.is_converse(summarization_llm),
         )
+        self.shortener = (
+            SummaryLengthPrompt.for_language(self.language).get_prompt()
+            | summarization_llm
+            | HTMLTagOutputParser(tag_names=["summary"], required_tags=["summary"])
+        )
         return prompt | summarization_llm | output_parser
 
     @measure_execution_time
@@ -509,7 +528,7 @@ def prepare_inputs(items: list[Post]) -> list[dict[str, Any]]:
                         (post, f"{SUMMARY_FAILED} ({problem}).")
                     )
                     continue
-                post.summary = finalized
+                post.summary = self._fit_length(post, summary_output, finalized)
                 post.one_liner = summary_output.one_liner
                 post.tags = summary_output.tags
                 post.urls = summary_output.urls
@@ -532,6 +551,67 @@ def prepare_inputs(items: list[Post]) -> list[dict[str, Any]]:
                 self.filtered_out_posts.append((post, f"{SUMMARY_INVALID}: {e}"))
         return succeeded
 
+    # The budget's upper end (KO characters, EN words), and how far past it a
+    # summary may run before it is sent back once to be shortened.
+    LENGTH_LIMIT: ClassVar[dict[Language, int]] = {
+        Language.KO: KO_LENGTH_BAND[1],
+        Language.EN: 2000,
+    }
+    LENGTH_TOLERANCE: ClassVar[float] = 1.1
+
+    def _length(self, summary_html: str) -> int:
+        text = visible_text(summary_html)
+        return len(text) if self.language is Language.KO else len(text.split())
+
+    @retry_with_backoff("summary-length")
+    def _shorten(self, summary_html: str, measured: int) -> dict[str, Any]:
+        limit = self.LENGTH_LIMIT[self.language]
+        return self.shortener.invoke(
+            {"summary": summary_html, "measured": measured, "limit": limit}
+        )
+
+    def _fit_length(self, post: Post, output: SummaryOutput, finalized: str) -> str:
+        """``finalized``, or a shortened rewrite if it runs well past the budget.
+
+        The rewrite is kept only if it is usable and shorter; any failure keeps
+        the original, since a long summary is a flaw and a lost one is not.
+        """
+        measured = self._length(finalized)
+        limit = self.LENGTH_LIMIT[self.language]
+        if measured <= limit * self.LENGTH_TOLERANCE:
+            return finalized
+        try:
+            revised = SummaryOutput.model_validate(
+                {"summary": self._shorten(output.summary, measured)["summary"]}
+            ).summary
+            shortened = self._finalize_summary(post, revised)
+        except Exception as e:
+            logger.warning(
+                "Shortening the %d-unit summary of '%s' failed (%s); keeping it.",
+                measured,
+                post.title,
+                e,
+            )
+            return finalized
+        revised_length = self._length(shortened)
+        if self._unusable_summary(shortened) or revised_length >= measured:
+            logger.warning(
+                "Shortened summary of '%s' is not usable or not shorter (%d -> %d); "
+                "keeping the original.",
+                post.title,
+                measured,
+                revised_length,
+            )
+            return finalized
+        logger.info(
+            "Summary of '%s' shortened from %d to %d (limit %d).",
+            post.title,
+            measured,
+            revised_length,
+            limit,
+        )
+        return shortened
+
     # Summary tags read as plain text. Not one_liner: _clean_one_liner already
     # decodes it once, and a second decode turned "List&lt;String&gt;" into
     # "List" on the card.

app/main.py

@@ -21,7 +21,7 @@
 )
 from app.adapters.bedrock import BedrockLanguageModelFactory
 from app.adapters.feeds import PostCollector
-from app.adapters.http import is_email_renderable_image
+from app.adapters.http import probe_email_image
 from app.adapters.rendering import NewsletterBuilder
 from app.application.alerts import (
     maybe_send_clipped_newsletter_alert,
@@ -83,7 +83,7 @@ def handler(event: dict[str, Any], context: Any) -> dict[str, int | str]:
             config,
             PostCollector.from_urls,
             models,
-            is_email_renderable_image,
+            probe_email_image,
             end_date_str=end_date_str,
             language=language,
         )