implement: Compile the set as the PDF generator does (t30)

Author: peterbjohnson

in2lambda/validation/pdf/__init__.py

@@ -122,13 +122,16 @@ def _compiled(fields: list[tuple[str, str]], images: list[str]) -> list[Problem]
             input=_marked_document(fields, available),
             capture_output=True,
             text=True,
+            # Not the locale's encoding: a set holding any non-ASCII character would
+            # then fail to even be handed over under, say, LC_ALL=C.
+            encoding="utf-8",
             cwd=work,
             timeout=_TIMEOUT,
         )
         if run.returncode:
             return [Problem(_SET, f"pandoc cannot read the set: {run.stderr.strip()}")]
 
-        latex = (work / "set.tex").read_text()
+        latex = (work / "set.tex").read_text(encoding="utf-8")
         run = subprocess.run(
             [
                 "xelatex",
@@ -140,6 +143,7 @@ def _compiled(fields: list[tuple[str, str]], images: list[str]) -> list[Problem]
             stdin=subprocess.DEVNULL,
             capture_output=True,
             text=True,
+            encoding="utf-8",
             cwd=work,
             timeout=_TIMEOUT,
         )

tests/test_validation.py

@@ -9,7 +9,10 @@
 on the ``Summer2025`` branch.
 """
 
+import os
 import shutil
+import subprocess
+import sys
 from pathlib import Path
 
 import pytest
@@ -133,6 +136,37 @@ def test_image_that_is_not_on_disk_is_reported(tmp_path: Path) -> None:
     ]
 
 
+@needs_compiler
+def test_non_ascii_is_compiled_whatever_the_locale() -> None:
+    """Exports are full of curly quotes, and containers are often not UTF-8 locales.
+
+    Run in a process of its own because the locale is read when Python starts.
+    """
+    script = (
+        "from in2lambda.api.set import Set\n"
+        "from in2lambda.validation import validate\n"
+        "question_set = Set()\n"
+        "question_set.add_question('Quotes', 'The rocket\\u2019s mass.')\n"
+        "print(len(validate(question_set)))\n"
+    )
+    run = subprocess.run(
+        [sys.executable, "-c", script],
+        env=os.environ
+        | {
+            "LC_ALL": "C",
+            "LANG": "C",
+            "PYTHONUTF8": "0",
+            "PYTHONCOERCECLOCALE": "0",
+            "PYTHONIOENCODING": "utf-8",
+        },
+        capture_output=True,
+        text=True,
+    )
+
+    assert run.returncode == 0, run.stderr
+    assert run.stdout.strip() == "0"
+
+
 def test_missing_compiler_says_what_to_install(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None: