diff --git a/evaluation_function/evaluation.py b/evaluation_function/evaluation.py index b9f4c18..5c42244 100755 --- a/evaluation_function/evaluation.py +++ b/evaluation_function/evaluation.py @@ -160,6 +160,57 @@ def _failure_result(message, include_feedback): result.add_feedback("feedback", message) return result +def _coerce_file_specs(raw: Any) -> list: + """Normalise a raw files value into a list of {url, name} dicts. + + Entries may already be dicts, or JSON-encoded strings — the LF web + client currently serialises each upload entry to a string. + """ + if not isinstance(raw, (list, tuple)): + return [] + specs = [] + for entry in raw: + if isinstance(entry, str): + try: + entry = json.loads(entry) + except (ValueError, TypeError): + continue + if isinstance(entry, dict): + specs.append(entry) + return specs + +def _unwrap_payload(value: Any) -> tuple[str, list]: + payload = value + if isinstance(payload, str): + try: + parsed = json.loads(payload) + except (ValueError, TypeError): + parsed = None + if isinstance(parsed, dict) and ("code" in parsed or "files" in parsed): + payload = parsed + + if isinstance(payload, dict): + return str(payload.get("code") or ""), _coerce_file_specs(payload.get("files")) + if isinstance(payload, str): + return payload, [] + return str(payload), [] + +def _resolve_submission(response: Any, params: Params) -> tuple[str, list]: + """Split the submission into (code, file_specs). + + Files listed in the response take precedence; params["files"] is the + fallback. + """ + code, response_files = _unwrap_payload(response) + file_specs = response_files or _coerce_file_specs(params.get("files")) + return code, file_specs + + +def _answer_code(answer: Any) -> str: + """The code string from the answer field, unwrapping a {code, files} + payload the same way the submission is unwrapped.""" + return _unwrap_payload(answer)[0] + def evaluation_function( response: Any, @@ -189,6 +240,9 @@ def evaluation_function( to output the evaluation response. """ + answer = _answer_code(answer) + response, _ = _resolve_submission(response, params) + client = OpenAI( api_key=os.environ.get("OPENROUTER_API_KEY"), base_url="https://openrouter.ai/api/v1",