From aabb7601b1e984ca202004f52b1cf3c4b61d4469 Mon Sep 17 00:00:00 2001
From: Marcus Messer <marcus@marcusm.co.uk>
Date: Fri, 25 Sep 2026 08:55:14 +0100
Subject: [PATCH 1/2] Add helper functions to normalize file specs and unwrap
 payloads in evaluation function

---
 evaluation_function/evaluation.py | 35 +++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/evaluation_function/evaluation.py b/evaluation_function/evaluation.py
index b9f4c18..2dde381 100755
--- a/evaluation_function/evaluation.py
+++ b/evaluation_function/evaluation.py
@@ -160,6 +160,41 @@ def _failure_result(message, include_feedback):
         result.add_feedback("feedback", message)
     return result
 
+def _coerce_file_specs(raw: Any) -> list:
+    """Normalise a raw files value into a list of {url, name} dicts.
+
+    Entries may already be dicts, or JSON-encoded strings — the LF web
+    client currently serialises each upload entry to a string.
+    """
+    if not isinstance(raw, (list, tuple)):
+        return []
+    specs = []
+    for entry in raw:
+        if isinstance(entry, str):
+            try:
+                entry = json.loads(entry)
+            except (ValueError, TypeError):
+                continue
+        if isinstance(entry, dict):
+            specs.append(entry)
+    return specs
+
+def _unwrap_payload(value: Any) -> tuple[str, list]:
+    payload = value
+    if isinstance(payload, str):
+        try:
+            parsed = json.loads(payload)
+        except (ValueError, TypeError):
+            parsed = None
+        if isinstance(parsed, dict) and ("code" in parsed or "files" in parsed):
+            payload = parsed
+
+    if isinstance(payload, dict):
+        return str(payload.get("code") or ""), _coerce_file_specs(payload.get("files"))
+    if isinstance(payload, str):
+        return payload, []
+    return str(payload), []
+
 
 def evaluation_function(
     response: Any,

From 3db8e8aa1bccc10bad277e52c4d5d2aa3bd2a567 Mon Sep 17 00:00:00 2001
From: Marcus Messer <marcus@marcusm.co.uk>
Date: Fri, 25 Sep 2026 09:05:44 +0100
Subject: [PATCH 2/2] Add helper functions to resolve submissions and extract
 code from answer payloads

---
 evaluation_function/evaluation.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/evaluation_function/evaluation.py b/evaluation_function/evaluation.py
index 2dde381..5c42244 100755
--- a/evaluation_function/evaluation.py
+++ b/evaluation_function/evaluation.py
@@ -195,6 +195,22 @@ def _unwrap_payload(value: Any) -> tuple[str, list]:
         return payload, []
     return str(payload), []
 
+def _resolve_submission(response: Any, params: Params) -> tuple[str, list]:
+    """Split the submission into (code, file_specs).
+
+    Files listed in the response take precedence; params["files"] is the
+    fallback.
+    """
+    code, response_files = _unwrap_payload(response)
+    file_specs = response_files or _coerce_file_specs(params.get("files"))
+    return code, file_specs
+
+
+def _answer_code(answer: Any) -> str:
+    """The code string from the answer field, unwrapping a {code, files}
+    payload the same way the submission is unwrapped."""
+    return _unwrap_payload(answer)[0]
+
 
 def evaluation_function(
     response: Any,
@@ -224,6 +240,9 @@ def evaluation_function(
     to output the evaluation response.
     """
 
+    answer = _answer_code(answer)
+    response, _ = _resolve_submission(response, params)
+
     client = OpenAI(
         api_key=os.environ.get("OPENROUTER_API_KEY"),
         base_url="https://openrouter.ai/api/v1",