From e5186d38e579a10bbb1056e90d1954cacdde0bdb Mon Sep 17 00:00:00 2001
From: Marcus Messer <marcus@marcusm.co.uk>
Date: Fri, 25 Sep 2026 09:16:49 +0100
Subject: [PATCH] Added functions for parsing and normalizing submission
 payloads and file specifications.

---
 app/evaluation.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/app/evaluation.py b/app/evaluation.py
index 6c56d20..ca6b46c 100755
--- a/app/evaluation.py
+++ b/app/evaluation.py
@@ -1,4 +1,6 @@
 import os
+
+import json
 from openai import OpenAI
 from dotenv import load_dotenv
 
@@ -16,6 +18,56 @@ def process_prompt(prompt, question, answer):
 
     return prompt
 
+def _coerce_file_specs(raw) -> list:
+    """Normalise a raw files value into a list of {url, name} dicts.
+
+    Entries may already be dicts, or JSON-encoded strings — the LF web
+    client currently serialises each upload entry to a string.
+    """
+    if not isinstance(raw, (list, tuple)):
+        return []
+    specs = []
+    for entry in raw:
+        if isinstance(entry, str):
+            try:
+                entry = json.loads(entry)
+            except (ValueError, TypeError):
+                continue
+        if isinstance(entry, dict):
+            specs.append(entry)
+    return specs
+
+def _unwrap_payload(value) -> tuple[str, list]:
+    payload = value
+    if isinstance(payload, str):
+        try:
+            parsed = json.loads(payload)
+        except (ValueError, TypeError):
+            parsed = None
+        if isinstance(parsed, dict) and ("code" in parsed or "files" in parsed):
+            payload = parsed
+
+    if isinstance(payload, dict):
+        return str(payload.get("code") or ""), _coerce_file_specs(payload.get("files"))
+    if isinstance(payload, str):
+        return payload, []
+    return str(payload), []
+
+def _resolve_submission(response, params) -> tuple[str, list]:
+    """Split the submission into (code, file_specs).
+
+    Files listed in the response take precedence; params["files"] is the
+    fallback.
+    """
+    code, response_files = _unwrap_payload(response)
+    file_specs = response_files or _coerce_file_specs(params.get("files"))
+    return code, file_specs
+
+
+def _answer_code(answer) -> str:
+    """The code string from the answer field, unwrapping a {code, files}
+    payload the same way the submission is unwrapped."""
+    return _unwrap_payload(answer)[0]
 
 def evaluation_function(response, answer, parameters):
     """
@@ -45,6 +97,9 @@ def evaluation_function(response, answer, parameters):
     to output the evaluation response.
     """
 
+    answer = _answer_code(answer)
+    response, _ = _resolve_submission(response, parameters)
+
     client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
 
     question = parameters.get("question")