`๋ก, ์์์ LaTeX๊ฐ ์๋๋ผ ์ผ๋ฐ ํ
์คํธ๋ ์ ๋์ฝ๋๋ก ์ฐ๊ฒ ํฉ๋๋ค.
@@ -1131,11 +1138,12 @@ lazy-loading์ฒ๋ผ ํ์๊ฐ ๋์น ์ด๋ฏธ์ง๋ ๋ ๋ฒ์งธ ์กฐ๊ฑด์ผ๋ก ์ด์
src(๋ชจ๋ธ์ด macOS ์คํฌ๋ฆฐ์ท ์ด๋ฆ์ U+202F๋ฅผ ์ผ๋ฐ ๊ณต๋ฐฑ์ผ๋ก ์ฎ๊ธด ๊ฒฝ์ฐ)๋ ์๋ฌธ URL๋ก ๋๋๋ฆฝ๋๋ค.
์ถ์ฒ๊ฐ ๋ง์๋ ๋ฉ์ผ์์ ๋ณด์ด์ง ์๋ ์ด๋ฏธ์ง๊ฐ ์์ต๋๋ค. `_drop_unrenderable_images`๋ ๋จ์
-์ด๋ฏธ์ง๋ฅผ ํ๋์ฉ ๋ฐ์ ๋ณด๊ณ (`http.is_email_renderable_image`, ์์ฝ๊ธฐ๋ ์์ฑ์ ์ธ์ `image_probe`๋ก ๋ฐ์ต๋๋ค), ์๋ต์ด PNGยทJPEGยทGIF๊ฐ ์๋๊ฑฐ๋
+์ด๋ฏธ์ง๋ฅผ ํ๋์ฉ ๋ฐ์ ๋ณด๊ณ (`http.probe_email_image`, ์์ฝ๊ธฐ๋ ์์ฑ์ ์ธ์ `image_probe`๋ก ๋ฐ์ต๋๋ค), ์๋ต์ด PNGยทJPEGยทGIF๊ฐ ์๋๊ฑฐ๋
๋ฐ์์ง์ง ์์ผ๋ฉด ์๋ฌธ์ด ๊ฐ์ ๊ทธ๋ฆผ์ ๋ํด ์ ์ด ๋ ๋ค๋ฅธ ํ๋ณธ(`srcset`, `data-src`, ``์
``)์ ์ฐจ๋ก๋ก ํ์ธํด ๋ฉ์ผ์์ ๋ณด์ด๋ ์ฒซ ํ๋ณธ์ผ๋ก ๋ฐ๊ฟ๋๋ค. ๋ฉ์ผ ํ๋ฉด์๋ 2๋ฐฐ ํด์๋๋ฉด ์ถฉ๋ถํ๋ฏ๋ก
1600px ์ดํ ์ค ๊ฐ์ฅ ํฐ ๊ฒ๋ถํฐ ๋ด
๋๋ค. ๋ง๋ ํ๋ณธ์ด ์์ผ๋ฉด figure๋ฅผ ์บก์
์งธ ์ง์๋๋ค. ํ์ธ์ ์ด๋ฏธ์ง๋น
-ํ ๋ฒ, 10์ด ์์ ํค๋๋ง ๋ฐ๊ณ (`IMAGE_PROBE_TIMEOUT_SECONDS`), ์์ฝ ํ๋๊ฐ ์ฐ๋ ํ์ธ ์๊ฐ์ ๋ชจ๋ ํฉ์ณ
+ํ ๋ฒ, 10์ด ์์ ์๋ต ํค๋์ ์ด๋ฏธ์ง ์๋ถ๋ถ 64KB๋ง ๋ฐ์ ๋๋น๋ฅผ ์ฝ๊ณ (`IMAGE_PROBE_TIMEOUT_SECONDS`,
+`IMAGE_HEADER_BYTES`), ์์ฝ ํ๋๊ฐ ์ฐ๋ ํ์ธ ์๊ฐ์ ๋ชจ๋ ํฉ์ณ
120์ด(`IMAGE_PROBE_BUDGET_SECONDS`)์
๋๋ค. ๊ทธ ์๊ฐ์ ๋๊ธฐ๋ฉด ์์ง ํ์ธํ์ง ๋ชปํ ์ด๋ฏธ์ง๋ ๋ณด๋ด์ง ์๊ณ
์ง์๋๋ค. Windows Outlook์ WebPยทAVIF ์๋ฆฌ์ alt ํ
์คํธ๋ฅผ
๋ณด์ฌ ์ฃผ๋ฏ๋ก, ๊ทธ๋๋ก ๋๋ฉด alt ํ
์คํธ์ ์บก์
์ด ํ ๋ฌธ์ฅ์ฒ๋ผ ์ด์ด์ ธ ๋ณด์
๋๋ค. URL ํ์ฅ์๋ก ์ถ์ธกํ์ง
@@ -1181,7 +1189,9 @@ SSRF ๊ฒ์ฌ๋ฅผ ๊ฑฐ์นฉ๋๋ค. ๋จ์ src๋ `_encode_image_srcs`๊ฐ URL์ ๊ทธ๋
ํ
๋๋ฆฌ์ ํจ๋ฉ์ ํ
ํ๋ฆฟ ๊ท์น์ด ์ค๋๋ค(ยง14). ๋ชจ๋ `
`์๋ `width="712"`์
`width:100%;max-width:712px;height:auto` ์ธ๋ผ์ธ ์คํ์ผ์ ๋ถ์
๋๋ค(`SUMMARY_IMAGE_WIDTH`, 800px
์ด์์ ์นด๋ ์ฌ๋ฐฑ๊ณผ ํจ๋ฉ์ ๋บ ๊ฐ). Windows Outlook์ `max-width`๋ฅผ ๋ฌด์ํ๋ฏ๋ก `width` ์์ฑ์ด
-์์ผ๋ฉด 2,000px ์ฐจํธ๊ฐ ์ด ์ ์ฒด๋ฅผ ์ฐฝ ๋ฐ์ผ๋ก ๋ฐ์ด๋
๋๋ค.
+์์ผ๋ฉด 2,000px ์ฐจํธ๊ฐ ์ด ์ ์ฒด๋ฅผ ์ฐฝ ๋ฐ์ผ๋ก ๋ฐ์ด๋
๋๋ค. ์ด๋ฏธ์ง ํ์ธ ๋จ๊ณ๊ฐ ์ค์ ๋๋น๋ฅผ ์ฝ์ด
+๊ทธ๋ณด๋ค ์ข์ ์ด๋ฏธ์ง์๋ ์๊ธฐ ๋๋น์ `max-width:100%;height:auto`๋ฅผ ๋ค์ ์ค๋๋ค. ๊ทธ๋์ ์์ ๊ทธ๋ฆผ์ด
+712px๋ก ๋์ด๋์ง ์์ต๋๋ค.
**ํ๋ฌธ์ผ๋ก ์ฝ๋ ํ๊ทธ.** ํํฐ ์๋ต์ `score`, `reason`๊ณผ ์์ฝ์ `tags`๋ ํ๋ฌธ์ผ๋ก
์ฝ์ต๋๋ค(`text_tags`). ํ๊ทธ ์์ HTML์ ์ง๋ ฌํํด ๋๋ ค์ฃผ๋ฉด "Q&A"๊ฐ "Q&A"๋ก ์ค๊ณ ,
diff --git a/docs/diagrams/pipeline-flow.png b/docs/diagrams/pipeline-flow.png
index 957a0b2..df698b9 100644
Binary files a/docs/diagrams/pipeline-flow.png and b/docs/diagrams/pipeline-flow.png differ
diff --git a/scripts/eval_filtering.py b/scripts/eval_filtering.py
index f098462..1604cb7 100644
--- a/scripts/eval_filtering.py
+++ b/scripts/eval_filtering.py
@@ -68,7 +68,7 @@ def _score_live(
os.environ.setdefault("CONFIG_FILE_SUFFIX", stage)
from app.adapters.bedrock import BedrockLanguageModelFactory
- from app.adapters.http import is_email_renderable_image
+ from app.adapters.http import probe_email_image
from app.application.summarize import Summarizer, SummarizerSettings
from app.configs import Config
from app.domain.constants import FilteringCriteria
@@ -91,7 +91,7 @@ def _score_live(
| {"filtering_criteria": FilteringCriteria(data.get("criteria", "all"))}
| ({"filtering_model_id": model} if model else {})
),
- is_email_renderable_image,
+ probe_email_image,
)
included = ", ".join(data.get("included_topics", []))
excluded = ", ".join(data.get("excluded_topics", []))
diff --git a/scripts/eval_summary_quality.py b/scripts/eval_summary_quality.py
index d4eceb2..aed5b45 100644
--- a/scripts/eval_summary_quality.py
+++ b/scripts/eval_summary_quality.py
@@ -150,7 +150,7 @@ def regenerate(date: str, stage: str, profile: str | None, suffix: str) -> str:
os.environ.setdefault("CONFIG_FILE_SUFFIX", stage)
from app.adapters.bedrock import BedrockLanguageModelFactory
- from app.adapters.http import is_email_renderable_image
+ from app.adapters.http import probe_email_image
from app.application.summarize import Summarizer, SummarizerSettings
from app.configs import Config
from app.domain.post import Post
@@ -174,7 +174,7 @@ def regenerate(date: str, stage: str, profile: str | None, suffix: str) -> str:
summarizer = Summarizer(
BedrockLanguageModelFactory(boto_session=session),
SummarizerSettings.model_validate(config.summarization.model_dump()),
- is_email_renderable_image,
+ probe_email_image,
)
posts = [
Post(
diff --git a/scripts/replay_filter.py b/scripts/replay_filter.py
index b22912c..3e6fcd3 100644
--- a/scripts/replay_filter.py
+++ b/scripts/replay_filter.py
@@ -121,7 +121,7 @@ def score(
import boto3
from app.adapters.bedrock import BedrockLanguageModelFactory
- from app.adapters.http import is_email_renderable_image
+ from app.adapters.http import probe_email_image
from app.application.summarize import Summarizer, SummarizerSettings
from app.domain.post import Post
from app.domain.summary import MODEL_FAILURE_PREFIXES
@@ -151,7 +151,7 @@ def score(
summarizer = Summarizer(
BedrockLanguageModelFactory(boto_session=session),
settings,
- is_email_renderable_image,
+ probe_email_image,
)
posts = [
Post(
diff --git a/tests/test_pipeline_guards.py b/tests/test_pipeline_guards.py
index dc3efba..d672841 100644
--- a/tests/test_pipeline_guards.py
+++ b/tests/test_pipeline_guards.py
@@ -451,7 +451,7 @@ class TestHandlerWiring:
def test_the_summarizer_gets_the_real_image_probe(self, monkeypatch):
# No fixture post carries an image, so a probe that dropped every image
# would pass the golden snapshot; pin the wiring instead.
- from app.adapters.http import is_email_renderable_image
+ from app.adapters.http import probe_email_image
from app.domain.crawl import CrawlReport
seen = {}
@@ -474,7 +474,7 @@ class resources:
monkeypatch.setattr(main, "log_run_summary", lambda *a: None)
monkeypatch.setattr(main, "fetch_and_filter_posts", _fetch)
assert main.handler({}, None)["statusCode"] == 200
- assert seen["probe"] is is_email_renderable_image
+ assert seen["probe"] is probe_email_image
class TestModelFailureAlertFromTheHandler:
diff --git a/tests/test_request_layer.py b/tests/test_request_layer.py
index b0d6199..9c947f2 100644
--- a/tests/test_request_layer.py
+++ b/tests/test_request_layer.py
@@ -197,11 +197,11 @@ def fake_getaddrinfo(host, *a, **k):
"url", ["http://169.254.169.254/x.png", "http://2130706433/x.png"]
)
def test_an_image_on_a_blocked_host_is_never_requested(self, monkeypatch, url):
- from app.adapters.http import is_email_renderable_image
+ from app.adapters.http import probe_email_image
session = _FakeSession([_FakeResponse(200)])
_patch_session(monkeypatch, session)
- assert not is_email_renderable_image(url)
+ assert probe_email_image(url) is None
assert session.calls == 0
def test_allows_hostname_resolving_to_public_ip(self, monkeypatch):
@@ -234,11 +234,17 @@ def respond(self, monkeypatch):
calls: list[dict] = []
- def install(content_type="image/png", ok=True, raises=None):
+ def install(content_type="image/png", ok=True, raises=None, body=b""):
class _Resp:
def __init__(self):
self.ok = ok
self.headers = {"content-type": content_type}
+ self.read = 0
+
+ def iter_content(self, chunk_size):
+ for start in range(0, len(body), chunk_size):
+ self.read += 1
+ yield body[start : start + chunk_size]
def __enter__(self):
return self
@@ -261,26 +267,76 @@ def fake(session, url, headers, timeout=None, stream=False):
"content_type", ["image/png", "image/jpeg; q=1", "image/gif"]
)
def test_email_safe_types_are_kept(self, respond, content_type):
- from app.adapters.http import is_email_renderable_image
+ from app.adapters.http import probe_email_image
respond(content_type)
- assert is_email_renderable_image("https://cdn.example.com/a")
+ # Kept, with an unknown width: the fake serves no image bytes.
+ assert probe_email_image("https://cdn.example.com/a") == 0
+
+ @pytest.mark.parametrize("fmt", ["PNG", "JPEG", "GIF"])
+ def test_the_width_comes_from_the_image_header(self, respond, fmt):
+ import io
+
+ from PIL import Image
+
+ from app.adapters.http import probe_email_image
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (120, 40)).save(buffer, format=fmt)
+ respond(f"image/{fmt.lower()}", body=buffer.getvalue())
+ assert probe_email_image("https://cdn.example.com/a") == 120
@pytest.mark.parametrize(
"content_type",
["image/webp", "image/avif", "image/svg+xml", "application/octet-stream"],
)
def test_types_mail_clients_do_not_show_are_dropped(self, respond, content_type):
- from app.adapters.http import is_email_renderable_image
+ from app.adapters.http import probe_email_image
respond(content_type)
- assert not is_email_renderable_image("https://cdn.example.com/a")
+ assert probe_email_image("https://cdn.example.com/a") is None
def test_an_image_that_does_not_load_is_dropped(self, respond):
- from app.adapters.http import is_email_renderable_image
+ from app.adapters.http import probe_email_image
respond(ok=False)
- assert not is_email_renderable_image("https://cdn.example.com/a")
+ assert probe_email_image("https://cdn.example.com/a") is None
+
+ def test_only_the_header_bytes_are_read(self, respond):
+ from app.adapters import http
+
+ respond(body=b"\x00" * (10 * http.IMAGE_HEADER_BYTES))
+ response_reads = []
+ original = http._image_width
+
+ def spy(response, deadline):
+ width = original(response, deadline)
+ response_reads.append(response.read)
+ return width
+
+ http._image_width = spy
+ try:
+ assert http.probe_email_image("https://cdn.example.com/a") == 0
+ finally:
+ http._image_width = original
+ assert response_reads == [http.IMAGE_HEADER_BYTES // (16 * 1024)]
+
+ def test_a_header_claiming_a_huge_image_is_kept_with_unknown_width(self, respond):
+ # Pillow refuses a header past its decompression-bomb limit; that must
+ # not drop the image, let alone the card around it.
+ import struct
+ import zlib
+
+ from app.adapters.http import probe_email_image
+
+ def chunk(kind: bytes, data: bytes) -> bytes:
+ crc = struct.pack(">I", zlib.crc32(kind + data))
+ return struct.pack(">I", len(data)) + kind + data + crc
+
+ ihdr = struct.pack(">IIBBBBB", 20000, 10000, 8, 2, 0, 0, 0)
+ png = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", b"")
+ respond(body=png)
+ assert probe_email_image("https://cdn.example.com/a") == 0
def test_one_short_header_only_attempt(self, respond):
# A hung host used to cost 3 attempts x 30 s per image, with the body
@@ -289,14 +345,14 @@ def test_one_short_header_only_attempt(self, respond):
from app.adapters.http import (
IMAGE_PROBE_TIMEOUT_SECONDS,
- is_email_renderable_image,
+ probe_email_image,
)
calls = respond(raises=requests.exceptions.ConnectTimeout("hung"))
- assert not is_email_renderable_image("https://cdn.example.com/a")
+ assert probe_email_image("https://cdn.example.com/a") is None
assert calls == [{"timeout": IMAGE_PROBE_TIMEOUT_SECONDS, "stream": True}]
def test_an_unsafe_scheme_is_never_fetched(self):
- from app.adapters.http import is_email_renderable_image
+ from app.adapters.http import probe_email_image
- assert not is_email_renderable_image("javascript:alert(1)")
+ assert probe_email_image("javascript:alert(1)") is None
diff --git a/tests/test_summarizer_internals.py b/tests/test_summarizer_internals.py
index 1f94457..d52f4e5 100644
--- a/tests/test_summarizer_internals.py
+++ b/tests/test_summarizer_internals.py
@@ -58,7 +58,7 @@ def _summarizer(min_score=0.7, **overrides):
s.language = s.settings.language
s.filtered_out_posts = []
# Every image loads; tests about the mail-client check pass their own probe.
- s.image_probe = lambda url: True
+ s.image_probe = lambda url: 0
from app.application.llm_batching import BatchProcessor
s.batch_processor = BatchProcessor(max_concurrency=2, batch_size=10)
@@ -339,7 +339,7 @@ def test_an_image_mail_clients_cannot_show_is_dropped_with_its_caption(
webp = "https://cdn.example.com/chart.width-100.format-webp.webp"
png = "https://cdn.example.com/arch.png"
s = _summarizer()
- s.image_probe = lambda url: url == png
+ s.image_probe = lambda url: 0 if url == png else None
post = _post("a", content=f'
')
post.images = [webp, png]
s.summarizer = _FakeChain(
@@ -365,7 +365,7 @@ def test_a_webp_is_swapped_for_a_png_rendition_the_article_lists(self, monkeypat
f'{base}-1536.png 1536w, {base}-2048.png 2048w">'
)
s = _summarizer()
- s.image_probe = lambda url: url.endswith(".png")
+ s.image_probe = lambda url: 0 if url.endswith(".png") else None
post = _post("a", content=source)
post.images = [f"{base}.webp"]
s.summarizer = _FakeChain(
@@ -391,12 +391,35 @@ def test_images_not_checked_within_the_budget_are_dropped(self, monkeypatch):
html, dropped = module._drop_unrenderable_images(
'
',
"",
- is_renderable=lambda url: True,
+ is_renderable=lambda url: 0,
budget_seconds=60,
clock=lambda: next(ticks),
)
assert "a.png" in html and dropped == ["https://x/b.png"]
+ @pytest.mark.parametrize(
+ ("width", "attr", "style"),
+ [
+ (120, "120", "max-width:100%;height:auto"),
+ (2369, "712", "width:100%;max-width:712px;height:auto"),
+ (0, "712", "width:100%;max-width:712px;height:auto"),
+ ],
+ )
+ def test_a_narrow_image_keeps_its_own_width(self, width, attr, style):
+ from bs4 import BeautifulSoup
+
+ from app.application import summarize as module
+ from app.domain.summary import SummaryOutput
+
+ sanitized = SummaryOutput.model_validate(
+ {"summary": 'x
', "tags": [], "urls": []}
+ ).summary
+ html, _ = module._drop_unrenderable_images(
+ sanitized, "", is_renderable=lambda url: width
+ )
+ img = BeautifulSoup(html, "html.parser").find("img")
+ assert (img["width"], img["style"]) == (attr, style)
+
def test_an_already_encoded_image_url_is_left_alone(self):
url = "https://cdn.example.com/a%20b.png?w=1200&q=90"
s = _summarizer()
@@ -964,3 +987,56 @@ def test_escaped_text_in_a_span_stays_escaped(self):
out = _postprocess_summary("Check `a < b` here.
")
assert out == "Check a < b here.
"
+
+
+class TestFitLength:
+ """An over-long summary is sent back once; nothing about it may lose the card."""
+
+ LONG = (
+ "" + "์ธก์ ๊ฐ๊ณผ ๋ฉ์ปค๋์ฆ์ ์ค๋ช
ํ๋ ๊ธด ๋ฌธ์ฅ์
๋๋ค. " * 330 + "
"
+ ) # ~7,600
+
+ class _Shortener:
+ def __init__(self, answer):
+ self.answer, self.calls = answer, []
+
+ def invoke(self, inputs):
+ self.calls.append(inputs)
+ if isinstance(self.answer, Exception):
+ raise self.answer
+ return {"summary": self.answer}
+
+ def _fit(self, monkeypatch, summary, answer):
+ from app.application import llm_batching
+ from app.domain.summary import SummaryOutput
+
+ monkeypatch.setattr(llm_batching.time, "sleep", lambda _s: None)
+ s = _summarizer()
+ s.shortener = self._Shortener(answer)
+ output = SummaryOutput.model_validate({"summary": summary})
+ finalized = s._finalize_summary(_post("a"), output.summary)
+ return s, s._fit_length(_post("a"), output, finalized)
+
+ def test_a_summary_within_tolerance_is_not_sent_back(self, monkeypatch):
+ s, out = self._fit(monkeypatch, BODY, "unused")
+ assert s.shortener.calls == [] and out.startswith("")
+
+ def test_an_over_long_summary_is_shortened_once(self, monkeypatch):
+ short = "
" + "์ค์ธ ์์ฝ ๋ฌธ์ฅ์
๋๋ค. " * 300 + "
"
+ s, out = self._fit(monkeypatch, self.LONG, short)
+ assert len(s.shortener.calls) == 1
+ assert s.shortener.calls[0]["limit"] == 6000
+ assert s.shortener.calls[0]["measured"] > 6600
+ assert "์ค์ธ ์์ฝ" in out
+
+ @pytest.mark.parametrize(
+ "answer",
+ [
+ ValueError("malformed"),
+ "" + "๋ ๊ธธ๊ฒ ๋๋ฆฐ ๋ฌธ์ฅ์
๋๋ค. " * 600 + "
",
+ "์งง์
", # below the usable floor
+ ],
+ )
+ def test_a_failed_or_worse_rewrite_keeps_the_original(self, monkeypatch, answer):
+ _, out = self._fit(monkeypatch, self.LONG, answer)
+ assert "์ธก์ ๊ฐ๊ณผ ๋ฉ์ปค๋์ฆ" in out