From 73de90504f1852d8fbe13da7e438b7765b1992df Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Tue, 25 Aug 2026 09:55:26 +0000 Subject: [PATCH 01/21] feat: implement evaluation task framework with feedback mechanism --- src/microbots/auto_memory/__init__.py | 8 + src/microbots/auto_memory/analyzer.py | 65 ++++++++ src/microbots/auto_memory/orchestrator.py | 124 ++++++++++++++ src/microbots/auto_memory/task.py | 189 ++++++++++++++++++++++ 4 files changed, 386 insertions(+) create mode 100644 src/microbots/auto_memory/__init__.py create mode 100644 src/microbots/auto_memory/analyzer.py create mode 100644 src/microbots/auto_memory/orchestrator.py create mode 100644 src/microbots/auto_memory/task.py diff --git a/src/microbots/auto_memory/__init__.py b/src/microbots/auto_memory/__init__.py new file mode 100644 index 0000000..27f9a4b --- /dev/null +++ b/src/microbots/auto_memory/__init__.py @@ -0,0 +1,8 @@ +"""Train <-> eval loop for repo-learning agents. + +Re-exports the public task, outcome, and orchestrator types used to define +an evaluation task and run it in a loop against a training agent. +""" + +from .task import CallbackResult, EvalOutcome, EvalTask +from .orchestrator import LoopResult, run_train_eval_loop \ No newline at end of file diff --git a/src/microbots/auto_memory/analyzer.py b/src/microbots/auto_memory/analyzer.py new file mode 100644 index 0000000..2589390 --- /dev/null +++ b/src/microbots/auto_memory/analyzer.py @@ -0,0 +1,65 @@ +"""Build feedback text for a failed evaluation round. + +Uses ``LogAnalysisBot`` to analyze the eval callback's raw log and produce +concrete feedback describing what went wrong, to be passed into the +training agent as ``feedback`` for the next round. +""" + +from logging import getLogger + +from microbots.auto_memory.task import EvalOutcome, EvalTask +from microbots.bot.LogAnalysisBot import LogAnalysisBot +from microbots.MicroBot import BotRunResult + +logger = getLogger(__name__) + +def build_feedback( + task: EvalTask, + outcome: EvalOutcome, + repo_path: str, + model: str, +) -> str: + """Analyze a failed eval outcome's log and produce training feedback. + + Parameters + ---------- + task : EvalTask + The eval task that produced ``outcome``. + outcome : EvalOutcome + The failed outcome to analyze, including its ``log_path``. + repo_path : str + Absolute path to the repo the task was evaluated against. + model : str + The model to use, in the format ``/``. + + Returns + ------- + str + Feedback text describing the root cause of the failure and what + the agent's memory notes should cover next time, suitable for + passing as ``feedback`` to ``run_training``. + """ + bot = LogAnalysisBot(model=model, folder_to_mount=repo_path) + result: BotRunResult = bot.run( + file_name=outcome.log_path, + user_prompt=( + "This log was produced while verifying whether an " + "automated agent completed its task correctly. Identify " + "the root cause of the failure and describe concretely " + "what the agent's memory notes should cover next time to " + "avoid this failure." + ), + ) + + if result.status and result.result: + return result.result + + logger.warning( + "LogAnalysisBot failed to analyze failure (%s); falling back to plain feedback", + result.error, + ) + return ( + f"Evaluation failed. Agent output: {outcome.output}\n" + f"Callback reason: {outcome.result.reason}" + ) + \ No newline at end of file diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py new file mode 100644 index 0000000..7558a51 --- /dev/null +++ b/src/microbots/auto_memory/orchestrator.py @@ -0,0 +1,124 @@ +"""Orchestrates the train <-> eval loop for repo-learning agents. + +Repeatedly runs an ``EvalTask`` against a repo, and on failure builds +feedback and retrains via the training agent, looping until the task +passes or ``max_rounds`` is exhausted. +""" + +from dataclasses import dataclass, field +from logging import getLogger +from pathlib import Path + +from microbots.auto_memory.analyzer import build_feedback +from microbots.auto_memory.task import EvalOutcome, EvalTask +from microbots.auto_memory.training.runner import run_training + +logger = getLogger(__name__) + +@dataclass +class LoopResult: + """Result of running ``run_train_eval_loop``. + + Attributes + ---------- + passed : bool + Whether the task passed within ``max_rounds``. + rounds_run : int + Number of eval rounds actually run. + final_outcome : EvalOutcome + The outcome of the last round run. + outcomes : list[EvalOutcome] + The outcome of every round run, in order. + """ + + passed: bool + rounds_run: int + final_outcome: EvalOutcome + outcomes: list[EvalOutcome] = field(default_factory=list) + +def run_train_eval_loop( + repo_path: str, + memory_dir: str, + model: str, + task: EvalTask, + max_rounds: int = 5, +) -> LoopResult: + """Run an eval task in a loop, retraining on failure until it passes. + + Each round runs ``task.run(...)``. If the task passes, the loop + returns immediately. If it fails, feedback is built from the round's + log and used to retrain via ``run_training`` before the next round. + The round's log file is always deleted before the next round starts. + + Parameters + ---------- + repo_path : str + Absolute path to the repo to evaluate and train against. + memory_dir : str + Directory where the training agent reads/writes memory files. + model : str + The model to use, in the format ``/``. + task : EvalTask + The eval task to run each round. + max_rounds : int + Maximum number of train/eval rounds to attempt. Defaults to 5. + + Returns + ------- + LoopResult + Whether the task passed, how many rounds ran, and every round's + outcome. + """ + outcomes: list[EvalOutcome] = [] + + for round_idx in range(max_rounds): + logger.info( + "run_train_eval_loop: round %d/%d starting", round_idx + 1, max_rounds + ) + outcome = task.run(repo_path, memory_dir, model) + outcomes.append(outcome) + + try: + if outcome.passed: + logger.info( + "run_train_eval_loop: passed on round %d/%d", round_idx + 1, max_rounds + ) + return LoopResult( + passed=True, + rounds_run=round_idx + 1, + final_outcome=outcome, + outcomes=outcomes, + ) + + logger.info( + "run_train_eval_loop: round %d failed (%s), retraining", + round_idx + 1, + outcome.result.reason, + ) + try: + feedback = build_feedback(task, outcome, repo_path, model) + + run_training( + repo_path=repo_path, + feedback=feedback, + memory_dir=memory_dir, + model=model, + ) + except Exception: + logger.exception( + "run_train_eval_loop: round %d failed to build feedback/retrain; " + "continuing to next round without retraining", + round_idx + 1, + ) + finally: + Path(outcome.log_path).unlink(missing_ok=True) + + logger.info( + "run_train_eval_loop: exhausted %d rounds without passing", max_rounds + ) + return LoopResult( + passed=False, + rounds_run=max_rounds, + final_outcome=outcomes[-1], + outcomes=outcomes, + ) \ No newline at end of file diff --git a/src/microbots/auto_memory/task.py b/src/microbots/auto_memory/task.py new file mode 100644 index 0000000..e0e8fef --- /dev/null +++ b/src/microbots/auto_memory/task.py @@ -0,0 +1,189 @@ +"""Defines the abstract eval task interface for the train <-> eval loop. + +An ``EvalTask`` describes one unit of work: how to prepare a repo, what +prompt to give the agent, how to verify the agent's output, and how to +clean up afterward. +""" + +import tempfile +from abc import ABC, abstractmethod +from dataclasses import dataclass +from logging import getLogger +from pathlib import Path + +from microbots.bot.WritingBot import WritingBot +from microbots.tools.MemoryTool import MemoryTool + +logger = getLogger(__name__) + +@dataclass +class CallbackResult: + """Result of verifying whether an eval task was completed correctly. + + Attributes + ---------- + passed : bool + Whether the agent's output satisfies the task's check. + reason : str + A short human-readable explanation of the pass/fail verdict. + """ + + passed: bool + reason: str + +@dataclass +class EvalOutcome: + """Full record of one eval round. + + Attributes + ---------- + passed : bool + Whether the round passed, mirrors ``result.passed``. + output : str | None + The agent's raw output for the round, if any. + result : CallbackResult + The verdict produced by ``EvalTask.check``. + log_path : str + Path to the round's log file, containing the agent output and + any failure/exception details recorded during the round. + """ + + passed: bool + output: str | None + result: CallbackResult + log_path: str + + +class EvalTask(ABC): + """Base class for a single evaluation task in the train <-> eval loop. + + Subclasses must implement ``build_prompt`` and ``check``, and may + override ``setup``, ``teardown``, and ``run`` as needed. + """ + + def setup(self, repo_path: str) -> None: + """Optional. Prepare repo/environment before the agent runs. + + Parameters + ---------- + repo_path : str + Absolute path to the repo to prepare. + """ + pass + + @abstractmethod + def build_prompt(self, repo_path: str) -> str: + """Required. Return the task prompt/instructions for the agent. + + Parameters + ---------- + repo_path : str + Absolute path to the repo the agent will operate on. + + Returns + ------- + str + The prompt/instructions to give the agent. + """ + + @abstractmethod + def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackResult: + """Required. Verify whether the task was actually completed correctly. + + Parameters + ---------- + repo_path : str + Absolute path to the repo the agent operated on. + agent_output : str + The agent's raw output/result text. + log_path : str + Path to a log file, already created by ``run``, that this + check may append verification details to. + + Returns + ------- + CallbackResult + The pass/fail verdict and its reason. + """ + + def teardown(self, repo_path: str) -> None: + """Optional. Clean up anything setup() created. + + Parameters + ---------- + repo_path : str + Absolute path to the repo that was prepared by ``setup``. + """ + pass + + def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: + """Default eval iteration: setup -> build_prompt -> WritingBot -> check -> teardown. + Override this entirely if your task needs a different bot type, + additional tools, or custom retry/orchestration logic. + + Parameters + ---------- + repo_path : str + Absolute path to the repo to run the eval round against. + memory_dir : str + Directory containing memory files to give the agent via + ``MemoryTool``. + model : str + The model to use, in the format ``/``. + + Returns + ------- + EvalOutcome + The result of this eval round, including the agent's output, + the check verdict, and the round's log file path. + """ + self.setup(repo_path) + log_path = tempfile.mktemp(suffix=".log") + Path(log_path).write_text("") + + try: + try: + prompt = self.build_prompt(repo_path) + bot = WritingBot( + model=model, + folder_to_mount=repo_path, + additional_tools=[MemoryTool(memory_dir=memory_dir)], + ) + bot_result = bot.run(prompt) + + with open(log_path, "a") as f: + f.write(f"Agent output:\n{bot_result.result}\n") + + if not bot_result.status: + reason = f"Bot run failed: {bot_result.error}" + with open(log_path, "a") as f: + f.write(f"\n{reason}\n") + result = CallbackResult(passed=False, reason=reason) + else: + result = self.check(repo_path, bot_result.result or "", log_path) + + return EvalOutcome( + passed=result.passed, + output=bot_result.result, + result=result, + log_path=log_path, + ) + except Exception as exc: + logger.exception( + "EvalTask.run: iteration raised %s", type(exc).__name__ + ) + with open(log_path, "a") as f: + f.write(f"\nException during eval iteration: {type(exc).__name__}: {exc}\n") + return EvalOutcome( + passed=False, + output=None, + result=CallbackResult( + passed=False, reason=f"{type(exc).__name__}: {exc}" + ), + log_path=log_path, + ) + finally: + try: + self.teardown(repo_path) + except Exception: + logger.exception("EvalTask.run: teardown() raised exception; ignoring") From c9d7b9b27940fff5f4fdb1968ba7d1399732d68b Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 26 Aug 2026 06:30:05 +0000 Subject: [PATCH 02/21] Add SweBenchVerifiedTask for evaluation and improve EvalTask interface --- src/microbots/auto_memory/analyzer.py | 2 +- .../auto_memory/eval_swebenchverified/eval.py | 269 ++++++++++++++++++ src/microbots/auto_memory/task.py | 8 +- 3 files changed, 274 insertions(+), 5 deletions(-) create mode 100644 src/microbots/auto_memory/eval_swebenchverified/eval.py diff --git a/src/microbots/auto_memory/analyzer.py b/src/microbots/auto_memory/analyzer.py index 2589390..dccea6a 100644 --- a/src/microbots/auto_memory/analyzer.py +++ b/src/microbots/auto_memory/analyzer.py @@ -44,7 +44,7 @@ def build_feedback( file_name=outcome.log_path, user_prompt=( "This log was produced while verifying whether an " - "automated agent completed its task correctly. Identify " + "agent completed its task correctly. Identify " "the root cause of the failure and describe concretely " "what the agent's memory notes should cover next time to " "avoid this failure." diff --git a/src/microbots/auto_memory/eval_swebenchverified/eval.py b/src/microbots/auto_memory/eval_swebenchverified/eval.py new file mode 100644 index 0000000..4c3ef15 --- /dev/null +++ b/src/microbots/auto_memory/eval_swebenchverified/eval.py @@ -0,0 +1,269 @@ +"""Minimal SWE-bench-verified eval task. + +Loads instances from the SWE-bench-verified dataset, checks out each +instance's repo at its base commit, has the agent attempt a fix, and +verifies the result via ``swebench.harness.run_evaluation``. +""" + +import json +import shutil +import subprocess +import sys +import tempfile +import uuid +from dataclasses import dataclass +from logging import getLogger +from pathlib import Path + +from datasets import load_dataset +import argparse +from microbots.auto_memory.task import CallbackResult, EvalTask +from microbots.auto_memory.orchestrator import run_train_eval_loop + +logger = getLogger(__name__) + +SWE_BENCH_SUITE = "SWE-bench/SWE-bench_Verified" + + +@dataclass +class SweBenchInstance: + """A single SWE-bench-verified dataset row. + + Attributes + ---------- + instance_id : str + Unique identifier for the instance, e.g. ``"django__django-11099"``. + repo : str + The GitHub repo this instance belongs to, e.g. ``"django/django"``. + base_commit : str + Commit hash representing the repo state before the issue's fix. + problem_statement : str + The GitHub issue title and body describing the bug to fix. + """ + + instance_id: str + repo: str + base_commit: str + problem_statement: str + + +def load_instances_of_repo( + dataset_name: str = SWE_BENCH_SUITE, + repo: str | None = None, +) -> list[SweBenchInstance]: + """Load all dataset instances, optionally filtered to a single repo. + + Parameters + ---------- + dataset_name : str + Hugging Face dataset name to load. Defaults to + ``SWE_BENCH_SUITE``. + repo : str | None + If given, only instances whose ``repo`` matches this value are + returned, e.g. ``"django/django"``. If ``None``, all instances + are returned. + + Returns + ------- + list[SweBenchInstance] + The matching instances. + """ + rows = load_dataset(dataset_name, split="test") + instances = [ + SweBenchInstance( + instance_id=row["instance_id"], + repo=row["repo"], + base_commit=row["base_commit"], + problem_statement=row["problem_statement"], + ) + for row in rows + if repo is None or row["repo"] == repo + ] + return instances + +def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_SUITE) -> SweBenchInstance: + """Load a single dataset instance by its instance ID. + + Parameters + ---------- + instance_id : str + The instance ID to look up, e.g. ``"django__django-11099"``. + dataset_name : str + Hugging Face dataset name to load. Defaults to + ``SWE_BENCH_SUITE``. + + Returns + ------- + SweBenchInstance + The matching instance. + + Raises + ------ + ValueError + If no instance with the given ``instance_id`` exists in the + dataset. + """ + rows = load_dataset(dataset_name, split="test") + for row in rows: + if row["instance_id"] == instance_id: + return SweBenchInstance( + instance_id=row["instance_id"], + repo=row["repo"], + base_commit=row["base_commit"], + problem_statement=row["problem_statement"], + ) + raise ValueError(f"instance_id not found: {instance_id}") + +class SweBenchVerifiedTask(EvalTask): + """Eval task that verifies a fix against one SWE-bench-verified instance. + + Checks out the instance's repo at its base commit, gives the agent + the issue's problem statement, and verifies the agent's patch using + the official SWE-bench evaluation harness. + + Parameters + ---------- + instance : SweBenchInstance + The dataset instance this task evaluates against. + """ + + def __init__(self, instance: SweBenchInstance): + """Initialize the task for a single dataset instance. + + Parameters + ---------- + instance : SweBenchInstance + The dataset instance this task evaluates against. + """ + self.instance = instance + + def setup(self, repo_path: str) -> None: + """Clone the instance's repo and check out its base commit. + + Parameters + ---------- + repo_path : str + Absolute path to clone the repo into. + """ + subprocess.run( + ["git", "clone", f"https://github.com/{self.instance.repo}.git", repo_path], + check=True, + ) + subprocess.run( + ["git", "checkout", self.instance.base_commit], cwd=repo_path, check=True + ) + + def build_prompt(self, repo_path: str) -> str: + """Return the instance's issue text as the agent's prompt. + + Parameters + ---------- + repo_path : str + Absolute path to the repo the agent will operate on. + + Returns + ------- + str + The instance's ``problem_statement``. + """ + return self.instance.problem_statement + + def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackResult: + """Verify the agent's patch using the SWE-bench evaluation harness. + + Captures the agent's changes as a git diff, submits it as a + prediction to ``swebench.harness.run_evaluation``, and checks + whether the harness marked this instance as resolved. + + Parameters + ---------- + repo_path : str + Absolute path to the repo the agent operated on. + agent_output : str + The agent's raw output/result text. Unused here, since + verification is based on the repo's git diff, not the + agent's textual output. + log_path : str + Path to a log file to append the harness's output to. + + Returns + ------- + CallbackResult + Whether the harness marked this instance as resolved. + """ + diff = subprocess.run( + ["git", "diff"], cwd=repo_path, capture_output=True, text=True + ).stdout + + run_id = f"microbots-{uuid.uuid4().hex[:8]}" + model_name_or_path = "microbots-eval-agent" + pred_path = Path(tempfile.mktemp(suffix=".json")) + report_dir = Path(tempfile.mkdtemp()) + pred_path.write_text(json.dumps([{ + "instance_id": self.instance.instance_id, + "model_patch": diff, + "model_name_or_path": model_name_or_path, + }])) + + try: + proc = subprocess.run( + [sys.executable, "-m", "swebench.harness.run_evaluation", + "--dataset_name", SWE_BENCH_SUITE, + "--max_workers", "1", + "--predictions_path", str(pred_path), + "--run_id", run_id, + "--report_dir", str(report_dir), + "--instance_ids", self.instance.instance_id], + #can add timeout if needed + capture_output=True, text=True, + cwd=report_dir, + ) + with open(log_path, "a") as f: + f.write(proc.stdout + proc.stderr) + + report_file = report_dir / f"{model_name_or_path}.{run_id}.json" + passed = False + if report_file.exists(): + report = json.loads(report_file.read_text()) + passed = self.instance.instance_id in report.get("resolved_ids", []) + finally: + pred_path.unlink(missing_ok=True) + shutil.rmtree(report_dir, ignore_errors=True) + + return CallbackResult(passed=passed, reason="resolved" if passed else "not resolved") + + def teardown(self, repo_path: str) -> None: + """Remove the cloned repo working directory. + + Parameters + ---------- + repo_path : str + Absolute path to the repo cloned by ``setup``. + """ + subprocess.run(["rm", "-rf", repo_path], check=False) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("--repo", help='e.g. "django/django"') + parser.add_argument("--instance-id", help='e.g. "django__django-11099"') + parser.add_argument("--model", default="azure-openai/gpt-5.5") + parser.add_argument("--max-rounds", type=int, default=5) + args = parser.parse_args() + + if args.instance_id: + instances = [load_instance_using_id(args.instance_id)] + else: + instances = load_instances_of_repo(repo=args.repo) + + for instance in instances: + task = SweBenchVerifiedTask(instance) + result = run_train_eval_loop( + repo_path=tempfile.mkdtemp(), + memory_dir="memory", + model=args.model, + task=task, + max_rounds=args.max_rounds, + ) + logger.info("%s: passed=%s", instance.instance_id, result.passed) diff --git a/src/microbots/auto_memory/task.py b/src/microbots/auto_memory/task.py index e0e8fef..1b4417c 100644 --- a/src/microbots/auto_memory/task.py +++ b/src/microbots/auto_memory/task.py @@ -57,19 +57,19 @@ class EvalOutcome: class EvalTask(ABC): """Base class for a single evaluation task in the train <-> eval loop. - Subclasses must implement ``build_prompt`` and ``check``, and may - override ``setup``, ``teardown``, and ``run`` as needed. + Subclasses must implement ``setup``, ``build_prompt``, and ``check``, + and may override ``teardown`` and ``run`` as needed. """ + @abstractmethod def setup(self, repo_path: str) -> None: - """Optional. Prepare repo/environment before the agent runs. + """Required. Prepare repo/environment before the agent runs. Parameters ---------- repo_path : str Absolute path to the repo to prepare. """ - pass @abstractmethod def build_prompt(self, repo_path: str) -> str: From 0070b6110044fcb26fc1acbd0021bb0d4604dbb1 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 26 Aug 2026 06:54:43 +0000 Subject: [PATCH 03/21] Add unit tests for evaluation tasks and orchestrator --- src/microbots/auto_memory/task.py | 2 +- .../eval_swebenchverified/test_eval.py | 243 ++++++++++++++++++ test/auto_memory/test_analyzer.py | 81 ++++++ test/auto_memory/test_orchestrator.py | 163 ++++++++++++ test/auto_memory/test_task.py | 192 ++++++++++++++ 5 files changed, 680 insertions(+), 1 deletion(-) create mode 100644 test/auto_memory/eval_swebenchverified/test_eval.py create mode 100644 test/auto_memory/test_analyzer.py create mode 100644 test/auto_memory/test_orchestrator.py create mode 100644 test/auto_memory/test_task.py diff --git a/src/microbots/auto_memory/task.py b/src/microbots/auto_memory/task.py index 1b4417c..8a1de46 100644 --- a/src/microbots/auto_memory/task.py +++ b/src/microbots/auto_memory/task.py @@ -12,7 +12,7 @@ from pathlib import Path from microbots.bot.WritingBot import WritingBot -from microbots.tools.MemoryTool import MemoryTool +from microbots.tools.tool_definitions.memory_tool import MemoryTool logger = getLogger(__name__) diff --git a/test/auto_memory/eval_swebenchverified/test_eval.py b/test/auto_memory/eval_swebenchverified/test_eval.py new file mode 100644 index 0000000..4115e45 --- /dev/null +++ b/test/auto_memory/eval_swebenchverified/test_eval.py @@ -0,0 +1,243 @@ +"""Unit tests for microbots.auto_memory.eval_swebenchverified.eval.""" + +import json +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/"))) + +from microbots.auto_memory.eval_swebenchverified.eval import ( + SweBenchInstance, + SweBenchVerifiedTask, + load_instance_using_id, + load_instances_of_repo, +) + +MODULE = "microbots.auto_memory.eval_swebenchverified.eval" + + +def _fake_rows(): + return [ + { + "instance_id": "django__django-1", + "repo": "django/django", + "base_commit": "abc123", + "problem_statement": "fix bug 1", + }, + { + "instance_id": "astropy__astropy-1", + "repo": "astropy/astropy", + "base_commit": "def456", + "problem_statement": "fix bug 2", + }, + { + "instance_id": "django__django-2", + "repo": "django/django", + "base_commit": "ghi789", + "problem_statement": "fix bug 3", + }, + ] + + +# --------------------------------------------------------------------------- +# load_instances_of_repo / load_instance_using_id +# --------------------------------------------------------------------------- + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instances_of_repo_filters_by_repo(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + instances = load_instances_of_repo(repo="django/django") + + assert [i.instance_id for i in instances] == ["django__django-1", "django__django-2"] + assert all(isinstance(i, SweBenchInstance) for i in instances) + + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instances_of_repo_returns_all_when_repo_none(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + instances = load_instances_of_repo(repo=None) + + assert len(instances) == 3 + + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instance_using_id_returns_matching_instance(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + instance = load_instance_using_id("astropy__astropy-1") + + assert instance.repo == "astropy/astropy" + assert instance.problem_statement == "fix bug 2" + + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instance_using_id_raises_when_not_found(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + with pytest.raises(ValueError, match="not found"): + load_instance_using_id("does-not-exist") + + +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.setup / build_prompt / teardown +# --------------------------------------------------------------------------- + +def _instance(): + return SweBenchInstance( + instance_id="django__django-1", + repo="django/django", + base_commit="abc123", + problem_statement="fix the bug", + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_setup_clones_and_checks_out_base_commit(mock_run): + task = SweBenchVerifiedTask(_instance()) + task.setup("/repo") + + clone_call, checkout_call = mock_run.call_args_list + assert clone_call.args[0] == ["git", "clone", "https://github.com/django/django.git", "/repo"] + assert checkout_call.args[0] == ["git", "checkout", "abc123"] + assert checkout_call.kwargs["cwd"] == "/repo" + + +@pytest.mark.unit +def test_build_prompt_returns_problem_statement(): + task = SweBenchVerifiedTask(_instance()) + assert task.build_prompt("/repo") == "fix the bug" + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_teardown_removes_repo_path(mock_run): + task = SweBenchVerifiedTask(_instance()) + task.teardown("/repo") + + mock_run.assert_called_once_with(["rm", "-rf", "/repo"], check=False) + + +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.check +# --------------------------------------------------------------------------- + +def _make_fake_subprocess_run(resolved: bool, raise_on_harness: bool = False): + """Build a subprocess.run stand-in that fakes git diff + the harness call.""" + + def _fake_run(cmd, **kwargs): + if cmd[:2] == ["git", "diff"]: + return MagicMock(stdout="diff --git a/x.py b/x.py\n+fix", stderr="", returncode=0) + if "swebench.harness.run_evaluation" in cmd: + if raise_on_harness: + raise RuntimeError("harness crashed") + run_id = cmd[cmd.index("--run_id") + 1] + report_dir = kwargs["cwd"] + instance_id = cmd[cmd.index("--instance_ids") + 1] + report = {"resolved_ids": [instance_id] if resolved else []} + (Path(report_dir) / f"microbots-eval-agent.{run_id}.json").write_text(json.dumps(report)) + return MagicMock(stdout="harness ran\n", stderr="", returncode=0) + return MagicMock(stdout="", stderr="", returncode=0) + + return _fake_run + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_passed_true_when_instance_in_resolved_ids(mock_run, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + result = task.check("/repo", "agent output", str(log_path)) + + assert result.passed is True + assert result.reason == "resolved" + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_passed_false_when_instance_not_in_resolved_ids(mock_run, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=False) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + result = task.check("/repo", "agent output", str(log_path)) + + assert result.passed is False + assert result.reason == "not resolved" + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_passed_false_when_report_file_never_written(mock_run, tmp_path): + # harness call succeeds but never writes a report file (e.g. it errored internally) + def _fake_run(cmd, **kwargs): + if cmd[:2] == ["git", "diff"]: + return MagicMock(stdout="diff", stderr="", returncode=0) + return MagicMock(stdout="", stderr="", returncode=1) + + mock_run.side_effect = _fake_run + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + result = task.check("/repo", "agent output", str(log_path)) + + assert result.passed is False + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_appends_to_log_file_without_truncating_existing_content(mock_run, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True) + log_path = tmp_path / "check.log" + log_path.write_text("Agent output:\nprevious content\n") + + task = SweBenchVerifiedTask(_instance()) + task.check("/repo", "agent output", str(log_path)) + + content = log_path.read_text() + assert "previous content" in content + assert "harness ran" in content + + +@pytest.mark.unit +@patch(f"{MODULE}.shutil.rmtree") +@patch(f"{MODULE}.subprocess.run") +def test_check_cleans_up_pred_path_and_report_dir_on_success(mock_run, mock_rmtree, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + task.check("/repo", "agent output", str(log_path)) + + mock_rmtree.assert_called_once() + + +@pytest.mark.unit +@patch(f"{MODULE}.shutil.rmtree") +@patch(f"{MODULE}.subprocess.run") +def test_check_cleans_up_even_when_harness_raises(mock_run, mock_rmtree, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True, raise_on_harness=True) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + with pytest.raises(RuntimeError, match="harness crashed"): + task.check("/repo", "agent output", str(log_path)) + + mock_rmtree.assert_called_once() diff --git a/test/auto_memory/test_analyzer.py b/test/auto_memory/test_analyzer.py new file mode 100644 index 0000000..601b044 --- /dev/null +++ b/test/auto_memory/test_analyzer.py @@ -0,0 +1,81 @@ +"""Unit tests for microbots.auto_memory.analyzer.""" + +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) + +from microbots.auto_memory.analyzer import build_feedback +from microbots.auto_memory.task import CallbackResult, EvalOutcome +from microbots.MicroBot import BotRunResult + + +def _make_outcome(reason: str = "tests failed", output: str = "agent output") -> EvalOutcome: + return EvalOutcome( + passed=False, + output=output, + result=CallbackResult(passed=False, reason=reason), + log_path="/tmp/some.log", + ) + + +@pytest.mark.unit +@patch("microbots.auto_memory.analyzer.LogAnalysisBot") +def test_build_feedback_returns_bot_result_on_success(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult( + status=True, result="root cause: missing edge case handling", error=None + ) + mock_bot_cls.return_value = mock_bot + + outcome = _make_outcome() + feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") + + assert feedback == "root cause: missing edge case handling" + mock_bot_cls.assert_called_once_with(model="azure-openai/gpt-4o", folder_to_mount="/repo") + mock_bot.run.assert_called_once() + assert mock_bot.run.call_args.kwargs["file_name"] == outcome.log_path + + +@pytest.mark.unit +@patch("microbots.auto_memory.analyzer.LogAnalysisBot") +def test_build_feedback_falls_back_when_bot_status_false(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed") + mock_bot_cls.return_value = mock_bot + + outcome = _make_outcome(reason="tests failed", output="some output") + feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") + + assert "some output" in feedback + assert "tests failed" in feedback + + +@pytest.mark.unit +@patch("microbots.auto_memory.analyzer.LogAnalysisBot") +def test_build_feedback_falls_back_when_result_is_empty(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="", error=None) + mock_bot_cls.return_value = mock_bot + + outcome = _make_outcome(reason="assertion error", output="agent tried X") + feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") + + assert "agent tried X" in feedback + assert "assertion error" in feedback + + +@pytest.mark.unit +@patch("microbots.auto_memory.analyzer.LogAnalysisBot") +def test_build_feedback_falls_back_when_result_is_none(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result=None, error=None) + mock_bot_cls.return_value = mock_bot + + outcome = _make_outcome() + feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") + + assert "Evaluation failed" in feedback diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py new file mode 100644 index 0000000..2bbd2d7 --- /dev/null +++ b/test/auto_memory/test_orchestrator.py @@ -0,0 +1,163 @@ +"""Unit tests for microbots.auto_memory.orchestrator.""" + +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) + +from microbots.auto_memory.orchestrator import LoopResult, run_train_eval_loop +from microbots.auto_memory.task import CallbackResult, EvalOutcome + + +def _make_outcome(passed: bool, log_path: str, reason: str = "reason") -> EvalOutcome: + return EvalOutcome( + passed=passed, + output="agent output", + result=CallbackResult(passed=passed, reason=reason), + log_path=log_path, + ) + + +def _touch(path: str) -> str: + Path(path).write_text("log contents") + return path + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, mock_run_training, tmp_path): + log_path = _touch(str(tmp_path / "round1.log")) + task = MagicMock() + task.run.return_value = _make_outcome(passed=True, log_path=log_path) + + result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + + assert isinstance(result, LoopResult) + assert result.passed is True + assert result.rounds_run == 1 + assert task.run.call_count == 1 + mock_build_feedback.assert_not_called() + mock_run_training.assert_not_called() + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, mock_run_training, tmp_path): + log1 = _touch(str(tmp_path / "round1.log")) + log2 = _touch(str(tmp_path / "round2.log")) + task = MagicMock() + task.run.side_effect = [ + _make_outcome(passed=False, log_path=log1), + _make_outcome(passed=True, log_path=log2), + ] + mock_build_feedback.return_value = "feedback text" + + result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + + assert result.passed is True + assert result.rounds_run == 2 + mock_build_feedback.assert_called_once() + mock_run_training.assert_called_once_with( + repo_path="/repo", feedback="feedback text", memory_dir="/memory", model="azure-openai/gpt-4o" + ) + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_training, tmp_path): + task = MagicMock() + task.run.side_effect = [ + _make_outcome(passed=False, log_path=_touch(str(tmp_path / f"round{i}.log"))) + for i in range(3) + ] + mock_build_feedback.return_value = "feedback text" + + result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=3) + + assert result.passed is False + assert result.rounds_run == 3 + assert len(result.outcomes) == 3 + assert result.final_outcome is result.outcomes[-1] + assert mock_build_feedback.call_count == 3 + assert mock_run_training.call_count == 3 + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_log_path_deleted_after_passing_round(mock_build_feedback, mock_run_training, tmp_path): + log_path = _touch(str(tmp_path / "round1.log")) + task = MagicMock() + task.run.return_value = _make_outcome(passed=True, log_path=log_path) + + run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + + assert not Path(log_path).exists() + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_log_path_deleted_after_failing_round(mock_build_feedback, mock_run_training, tmp_path): + log1 = _touch(str(tmp_path / "round1.log")) + log2 = _touch(str(tmp_path / "round2.log")) + task = MagicMock() + task.run.side_effect = [ + _make_outcome(passed=False, log_path=log1), + _make_outcome(passed=True, log_path=log2), + ] + mock_build_feedback.return_value = "feedback text" + + run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + + assert not Path(log1).exists() + assert not Path(log2).exists() + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_run_training, tmp_path): + log1 = _touch(str(tmp_path / "round1.log")) + log2 = _touch(str(tmp_path / "round2.log")) + task = MagicMock() + task.run.side_effect = [ + _make_outcome(passed=False, log_path=log1), + _make_outcome(passed=True, log_path=log2), + ] + mock_build_feedback.side_effect = RuntimeError("analysis bot crashed") + + result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + + assert result.passed is True + assert result.rounds_run == 2 + mock_run_training.assert_not_called() + assert not Path(log1).exists() + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_run_training, tmp_path): + log1 = _touch(str(tmp_path / "round1.log")) + log2 = _touch(str(tmp_path / "round2.log")) + task = MagicMock() + task.run.side_effect = [ + _make_outcome(passed=False, log_path=log1), + _make_outcome(passed=True, log_path=log2), + ] + mock_build_feedback.return_value = "feedback text" + mock_run_training.side_effect = RuntimeError("training crashed") + + result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + + assert result.passed is True + assert result.rounds_run == 2 + assert not Path(log1).exists() diff --git a/test/auto_memory/test_task.py b/test/auto_memory/test_task.py new file mode 100644 index 0000000..bfc2cb6 --- /dev/null +++ b/test/auto_memory/test_task.py @@ -0,0 +1,192 @@ +"""Unit tests for microbots.auto_memory.task.""" + +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) + +from microbots.auto_memory.task import CallbackResult, EvalOutcome, EvalTask +from microbots.MicroBot import BotRunResult + + +class _StubTask(EvalTask): + """A minimal concrete EvalTask used to exercise the base run() logic.""" + + def __init__(self, check_result=None, check_side_effect=None, build_prompt_side_effect=None): + self.setup_calls = [] + self.teardown_calls = [] + self.check_calls = [] + self._check_result = check_result or CallbackResult(passed=True, reason="ok") + self._check_side_effect = check_side_effect + self._build_prompt_side_effect = build_prompt_side_effect + + def setup(self, repo_path): + self.setup_calls.append(repo_path) + + def build_prompt(self, repo_path): + if self._build_prompt_side_effect: + raise self._build_prompt_side_effect + return "do the task" + + def check(self, repo_path, agent_output, log_path): + self.check_calls.append((repo_path, agent_output, log_path)) + if self._check_side_effect: + raise self._check_side_effect + return self._check_result + + def teardown(self, repo_path): + self.teardown_calls.append(repo_path) + + +class _RaisingTeardownTask(_StubTask): + def teardown(self, repo_path): + super().teardown(repo_path) + raise RuntimeError("teardown boom") + + +class _DefaultTeardownTask(EvalTask): + """A task that relies on EvalTask's default no-op teardown.""" + + def setup(self, repo_path): + pass + + def build_prompt(self, repo_path): + return "do the task" + + def check(self, repo_path, agent_output, log_path): + return CallbackResult(passed=True, reason="ok") + + +@pytest.mark.unit +def test_setup_and_check_are_abstract(): + with pytest.raises(TypeError): + EvalTask() + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock_memory_tool): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="agent did stuff", error=None) + mock_bot_cls.return_value = mock_bot + + task = _StubTask() + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert task.setup_calls == ["/repo"] + assert task.check_calls == [("/repo", "agent did stuff", outcome.log_path)] + assert task.teardown_calls == ["/repo"] + assert outcome.passed is True + assert outcome.output == "agent did stuff" + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_tool): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) + mock_bot_cls.return_value = mock_bot + + seen_log_exists = {} + + class _CheckingTask(_StubTask): + def check(self, repo_path, agent_output, log_path): + seen_log_exists["exists"] = os.path.exists(log_path) + return super().check(repo_path, agent_output, log_path) + + task = _CheckingTask() + task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert seen_log_exists["exists"] is True + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed") + mock_bot_cls.return_value = mock_bot + + task = _StubTask() + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert task.check_calls == [] + assert outcome.passed is False + assert "bot crashed" in outcome.result.reason + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_converts_build_prompt_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): + task = _StubTask(build_prompt_side_effect=ValueError("bad prompt")) + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert outcome.passed is False + assert "bad prompt" in outcome.result.reason + with open(outcome.log_path) as f: + assert "bad prompt" in f.read() + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) + mock_bot_cls.return_value = mock_bot + + task = _StubTask(check_side_effect=RuntimeError("check exploded")) + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert outcome.passed is False + assert "check exploded" in outcome.result.reason + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_still_calls_teardown_when_body_raises(mock_bot_cls, mock_memory_tool): + mock_bot_cls.side_effect = RuntimeError("bot construction failed") + + task = _StubTask() + task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert task.teardown_calls == ["/repo"] + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_teardown_exception_does_not_clobber_returned_outcome(mock_bot_cls, mock_memory_tool): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) + mock_bot_cls.return_value = mock_bot + + task = _RaisingTeardownTask() + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + # teardown() raised, but the already-computed EvalOutcome must still be returned + assert isinstance(outcome, EvalOutcome) + assert outcome.passed is True + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_uses_default_noop_teardown_when_not_overridden(mock_bot_cls, mock_memory_tool): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) + mock_bot_cls.return_value = mock_bot + + task = _DefaultTeardownTask() + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert outcome.passed is True + From 90ed58656c84a099198e3b26d8d633ca9d63dfd6 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 26 Aug 2026 09:20:18 +0000 Subject: [PATCH 04/21] refactor: update training function references to use run_training_loop and add training_iterations parameter --- src/microbots/auto_memory/orchestrator.py | 14 +++-- test/auto_memory/test_orchestrator.py | 62 +++++++++++++++-------- 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index 7558a51..8f1dcfb 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -11,7 +11,7 @@ from microbots.auto_memory.analyzer import build_feedback from microbots.auto_memory.task import EvalOutcome, EvalTask -from microbots.auto_memory.training.runner import run_training +from microbots.auto_memory.training.runner import run_training_loop logger = getLogger(__name__) @@ -42,13 +42,15 @@ def run_train_eval_loop( model: str, task: EvalTask, max_rounds: int = 5, + training_iterations: int = 1, ) -> LoopResult: """Run an eval task in a loop, retraining on failure until it passes. Each round runs ``task.run(...)``. If the task passes, the loop returns immediately. If it fails, feedback is built from the round's - log and used to retrain via ``run_training`` before the next round. - The round's log file is always deleted before the next round starts. + log and used to retrain via ``run_training_loop`` before the next + round. The round's log file is always deleted before the next round + starts. Parameters ---------- @@ -62,6 +64,9 @@ def run_train_eval_loop( The eval task to run each round. max_rounds : int Maximum number of train/eval rounds to attempt. Defaults to 5. + training_iterations : int + Number of training passes to run per retraining round, each + reusing the same ``memory_dir``. Defaults to 1. Returns ------- @@ -98,11 +103,12 @@ def run_train_eval_loop( try: feedback = build_feedback(task, outcome, repo_path, model) - run_training( + run_training_loop( repo_path=repo_path, feedback=feedback, memory_dir=memory_dir, model=model, + iterations=training_iterations, ) except Exception: logger.exception( diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py index 2bbd2d7..ac21773 100644 --- a/test/auto_memory/test_orchestrator.py +++ b/test/auto_memory/test_orchestrator.py @@ -28,9 +28,9 @@ def _touch(path: str) -> str: @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, mock_run_training, tmp_path): +def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, mock_run_training_loop, tmp_path): log_path = _touch(str(tmp_path / "round1.log")) task = MagicMock() task.run.return_value = _make_outcome(passed=True, log_path=log_path) @@ -42,13 +42,13 @@ def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, m assert result.rounds_run == 1 assert task.run.call_count == 1 mock_build_feedback.assert_not_called() - mock_run_training.assert_not_called() + mock_run_training_loop.assert_not_called() @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, mock_run_training, tmp_path): +def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) task = MagicMock() @@ -63,15 +63,15 @@ def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, assert result.passed is True assert result.rounds_run == 2 mock_build_feedback.assert_called_once() - mock_run_training.assert_called_once_with( - repo_path="/repo", feedback="feedback text", memory_dir="/memory", model="azure-openai/gpt-4o" + mock_run_training_loop.assert_called_once_with( + repo_path="/repo", feedback="feedback text", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=1 ) @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_training, tmp_path): +def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_training_loop, tmp_path): task = MagicMock() task.run.side_effect = [ _make_outcome(passed=False, log_path=_touch(str(tmp_path / f"round{i}.log"))) @@ -86,13 +86,13 @@ def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_ assert len(result.outcomes) == 3 assert result.final_outcome is result.outcomes[-1] assert mock_build_feedback.call_count == 3 - assert mock_run_training.call_count == 3 + assert mock_run_training_loop.call_count == 3 @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_log_path_deleted_after_passing_round(mock_build_feedback, mock_run_training, tmp_path): +def test_log_path_deleted_after_passing_round(mock_build_feedback, mock_run_training_loop, tmp_path): log_path = _touch(str(tmp_path / "round1.log")) task = MagicMock() task.run.return_value = _make_outcome(passed=True, log_path=log_path) @@ -103,9 +103,9 @@ def test_log_path_deleted_after_passing_round(mock_build_feedback, mock_run_trai @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_log_path_deleted_after_failing_round(mock_build_feedback, mock_run_training, tmp_path): +def test_log_path_deleted_after_failing_round(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) task = MagicMock() @@ -122,9 +122,9 @@ def test_log_path_deleted_after_failing_round(mock_build_feedback, mock_run_trai @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_run_training, tmp_path): +def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) task = MagicMock() @@ -138,14 +138,14 @@ def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_ assert result.passed is True assert result.rounds_run == 2 - mock_run_training.assert_not_called() + mock_run_training_loop.assert_not_called() assert not Path(log1).exists() @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_run_training, tmp_path): +def test_loop_forwards_training_iterations_to_run_training_loop(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) task = MagicMock() @@ -154,7 +154,29 @@ def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_ru _make_outcome(passed=True, log_path=log2), ] mock_build_feedback.return_value = "feedback text" - mock_run_training.side_effect = RuntimeError("training crashed") + + run_train_eval_loop( + "/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5, training_iterations=4 + ) + + mock_run_training_loop.assert_called_once_with( + repo_path="/repo", feedback="feedback text", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=4 + ) + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training_loop") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_run_training_loop, tmp_path): + log1 = _touch(str(tmp_path / "round1.log")) + log2 = _touch(str(tmp_path / "round2.log")) + task = MagicMock() + task.run.side_effect = [ + _make_outcome(passed=False, log_path=log1), + _make_outcome(passed=True, log_path=log2), + ] + mock_build_feedback.return_value = "feedback text" + mock_run_training_loop.side_effect = RuntimeError("training crashed") result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) From b2b08ff8ca6a569c2ab88eddca67c1f047c06dd5 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Mon, 31 Aug 2026 05:07:34 +0000 Subject: [PATCH 05/21] modify file structure --- src/microbots/auto_memory/analyzer.py | 2 +- .../{eval_swebenchverified/eval.py => eval/swebenchverified.py} | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) rename src/microbots/auto_memory/{eval_swebenchverified/eval.py => eval/swebenchverified.py} (99%) diff --git a/src/microbots/auto_memory/analyzer.py b/src/microbots/auto_memory/analyzer.py index dccea6a..83e75cf 100644 --- a/src/microbots/auto_memory/analyzer.py +++ b/src/microbots/auto_memory/analyzer.py @@ -12,7 +12,7 @@ from microbots.MicroBot import BotRunResult logger = getLogger(__name__) - +#make this abstract def build_feedback( task: EvalTask, outcome: EvalOutcome, diff --git a/src/microbots/auto_memory/eval_swebenchverified/eval.py b/src/microbots/auto_memory/eval/swebenchverified.py similarity index 99% rename from src/microbots/auto_memory/eval_swebenchverified/eval.py rename to src/microbots/auto_memory/eval/swebenchverified.py index 4c3ef15..d83a2e0 100644 --- a/src/microbots/auto_memory/eval_swebenchverified/eval.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -249,6 +249,7 @@ def teardown(self, repo_path: str) -> None: parser.add_argument("--repo", help='e.g. "django/django"') parser.add_argument("--instance-id", help='e.g. "django__django-11099"') parser.add_argument("--model", default="azure-openai/gpt-5.5") + parser.add_argument("--max-rounds", type=int, default=5) args = parser.parse_args() From a3237ae2babeffe8c46fa863c78d469c8086bd07 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Mon, 31 Aug 2026 11:04:59 +0000 Subject: [PATCH 06/21] Implement task registry for EvalTask instances --- src/microbots/auto_memory/cli.py | 90 ++++ .../auto_memory/eval/swebenchverified.py | 147 ++++-- src/microbots/auto_memory/orchestrator.py | 49 +- src/microbots/auto_memory/task.py | 97 +--- src/microbots/auto_memory/task_registry.py | 92 ++++ .../auto_memory/eval/test_swebenchverified.py | 477 ++++++++++++++++++ .../eval_swebenchverified/test_eval.py | 243 --------- test/auto_memory/test_cli.py | 108 ++++ test/auto_memory/test_orchestrator.py | 36 +- test/auto_memory/test_task.py | 180 +------ test/auto_memory/test_task_registry.py | 104 ++++ 11 files changed, 1119 insertions(+), 504 deletions(-) create mode 100644 src/microbots/auto_memory/cli.py create mode 100644 src/microbots/auto_memory/task_registry.py create mode 100644 test/auto_memory/eval/test_swebenchverified.py delete mode 100644 test/auto_memory/eval_swebenchverified/test_eval.py create mode 100644 test/auto_memory/test_cli.py create mode 100644 test/auto_memory/test_task_registry.py diff --git a/src/microbots/auto_memory/cli.py b/src/microbots/auto_memory/cli.py new file mode 100644 index 0000000..98d3147 --- /dev/null +++ b/src/microbots/auto_memory/cli.py @@ -0,0 +1,90 @@ +"""Command-line entry point for the auto-memory train/eval loop. + +Two modes, selected by ``--task``: + +- ``--task `` given: run the full train <-> eval loop for that + task (via ``run_train_eval_loop``). +- ``--task`` omitted: train only, no eval task (via ``run_training_loop``, + with empty feedback). +""" + +import argparse +import logging + +from microbots.auto_memory.orchestrator import run_train_eval_loop, run_training_loop +from microbots.auto_memory.task_registry import TASK_REGISTRY, discover_tasks + +logger = logging.getLogger(__name__) + +# Import every task module so their @register_task decorators fire. +discover_tasks() + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse CLI args, including task-specific args when ``--task`` is given. + + Parameters + ---------- + argv : list[str] | None + Args to parse. Defaults to ``sys.argv[1:]`` when ``None``. + + Returns + ------- + argparse.Namespace + The parsed args. + """ + parser = argparse.ArgumentParser(description="Run the auto-memory train/eval loop.") + parser.add_argument("--repo", required=True, help="Absolute path to the repo.") + parser.add_argument("--memory-dir", required=True, help="Directory for memory files.") + parser.add_argument("--model", required=True, help='Model, e.g. "azure-openai/gpt-5.5".') + parser.add_argument( + "--task", + choices=sorted(TASK_REGISTRY), + help="Eval task to run. Omit to only run training, with no eval task.", + ) + parser.add_argument("--max-rounds", type=int, default=5) + parser.add_argument("--training-iterations", type=int, default=1) + + # First pass just to discover --task, so we can register its + # task-specific flags before the real parse. + known_args, _ = parser.parse_known_args(argv) + if known_args.task: + TASK_REGISTRY[known_args.task].add_cli_args(parser) + + return parser.parse_args(argv) + +def main(argv: list[str] | None = None) -> None: + """CLI entry point: run training only, or the full train/eval loop. + + Parameters + ---------- + argv : list[str] | None + Args to parse. Defaults to ``sys.argv[1:]`` when ``None``. + """ + args = parse_args(argv) + + if not args.task: + run_training_loop( + repo_path=args.repo, + feedback="", + memory_dir=args.memory_dir, + model=args.model, + iterations=args.training_iterations, + ) + return + + task_cls = TASK_REGISTRY[args.task] + for task in task_cls.from_cli_args(args): + result = run_train_eval_loop( + repo_path=args.repo, + memory_dir=args.memory_dir, + model=args.model, + task=task, + max_rounds=args.max_rounds, + training_iterations=args.training_iterations, + ) + logger.info( + "task=%s passed=%s rounds_run=%d", args.task, result.passed, result.rounds_run + ) + +if __name__ == "__main__": + main() diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index d83a2e0..2cd974b 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -1,10 +1,11 @@ -"""Minimal SWE-bench-verified eval task. +"""SWE-bench-verified eval task. Loads instances from the SWE-bench-verified dataset, checks out each instance's repo at its base commit, has the agent attempt a fix, and verifies the result via ``swebench.harness.run_evaluation``. """ +import argparse import json import shutil import subprocess @@ -16,9 +17,10 @@ from pathlib import Path from datasets import load_dataset -import argparse -from microbots.auto_memory.task import CallbackResult, EvalTask -from microbots.auto_memory.orchestrator import run_train_eval_loop +from microbots.auto_memory.task import CallbackResult, EvalOutcome, EvalTask +from microbots.auto_memory.task_registry import register_task +from microbots.bot.WritingBot import WritingBot +from microbots.tools.tool_definitions.memory_tool import MemoryTool logger = getLogger(__name__) @@ -114,6 +116,7 @@ def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_SUITE ) raise ValueError(f"instance_id not found: {instance_id}") +@register_task("swebenchverified") class SweBenchVerifiedTask(EvalTask): """Eval task that verifies a fix against one SWE-bench-verified instance. @@ -137,6 +140,47 @@ def __init__(self, instance: SweBenchInstance): """ self.instance = instance + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser) -> None: + """Register this task's CLI flags on ``parser``. + + Parameters + ---------- + parser : argparse.ArgumentParser + The CLI's argument parser to add task-specific flags to. + """ + parser.add_argument( + "--instance-id", + help='SWE-bench-verified instance ID, e.g. "django__django-11099".', + ) + parser.add_argument( + "--swebench-repo", + help='Restrict to instances for this repo, e.g. "django/django". ' + "Ignored if --instance-id is given.", + ) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace) -> list["SweBenchVerifiedTask"]: + """Build task(s) from parsed CLI args. + + Parameters + ---------- + args : argparse.Namespace + Parsed CLI args, expected to include ``instance_id`` and/or + ``swebench_repo`` (see ``add_cli_args``). + + Returns + ------- + list[SweBenchVerifiedTask] + One task per matching dataset instance. A single-element + list when ``--instance-id`` is given. + """ + if getattr(args, "instance_id", None): + instances = [load_instance_using_id(args.instance_id)] + else: + instances = load_instances_of_repo(repo=getattr(args, "swebench_repo", None)) + return [cls(instance) for instance in instances] + def setup(self, repo_path: str) -> None: """Clone the instance's repo and check out its base commit. @@ -242,29 +286,74 @@ def teardown(self, repo_path: str) -> None: """ subprocess.run(["rm", "-rf", repo_path], check=False) + def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: + """Run one eval iteration: setup -> build_prompt -> WritingBot -> check -> teardown. + + Parameters + ---------- + repo_path : str + Absolute path to the repo to run the eval round against. + memory_dir : str + Directory containing memory files to give the agent via + ``MemoryTool``. + model : str + The model to use, in the format ``/``. + + Returns + ------- + EvalOutcome + The result of this eval round, including the agent's output, + the check verdict, and the round's log file path. + """ + self.setup(repo_path) + log_path = tempfile.mktemp(suffix=".log") + Path(log_path).write_text("") + + try: + try: + prompt = self.build_prompt(repo_path) + bot = WritingBot( + model=model, + folder_to_mount=repo_path, + additional_tools=[MemoryTool(memory_dir=memory_dir)], + ) + bot_result = bot.run(prompt) + + with open(log_path, "a") as f: + f.write(f"Agent output:\n{bot_result.result}\n") + + if not bot_result.status: + reason = f"Bot run failed: {bot_result.error}" + with open(log_path, "a") as f: + f.write(f"\n{reason}\n") + result = CallbackResult(passed=False, reason=reason) + else: + result = self.check(repo_path, bot_result.result or "", log_path) + + return EvalOutcome( + passed=result.passed, + output=bot_result.result, + result=result, + log_path=log_path, + ) + except Exception as exc: + logger.exception( + "SweBenchVerifiedTask.run: iteration raised %s", type(exc).__name__ + ) + with open(log_path, "a") as f: + f.write(f"\nException during eval iteration: {type(exc).__name__}: {exc}\n") + return EvalOutcome( + passed=False, + output=None, + result=CallbackResult( + passed=False, reason=f"{type(exc).__name__}: {exc}" + ), + log_path=log_path, + ) + finally: + try: + self.teardown(repo_path) + except Exception: + logger.exception("SweBenchVerifiedTask.run: teardown() raised exception; ignoring") + -if __name__ == "__main__": - - parser = argparse.ArgumentParser() - parser.add_argument("--repo", help='e.g. "django/django"') - parser.add_argument("--instance-id", help='e.g. "django__django-11099"') - parser.add_argument("--model", default="azure-openai/gpt-5.5") - - parser.add_argument("--max-rounds", type=int, default=5) - args = parser.parse_args() - - if args.instance_id: - instances = [load_instance_using_id(args.instance_id)] - else: - instances = load_instances_of_repo(repo=args.repo) - - for instance in instances: - task = SweBenchVerifiedTask(instance) - result = run_train_eval_loop( - repo_path=tempfile.mkdtemp(), - memory_dir="memory", - model=args.model, - task=task, - max_rounds=args.max_rounds, - ) - logger.info("%s: passed=%s", instance.instance_id, result.passed) diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index 8f1dcfb..f4ca198 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -11,7 +11,7 @@ from microbots.auto_memory.analyzer import build_feedback from microbots.auto_memory.task import EvalOutcome, EvalTask -from microbots.auto_memory.training.runner import run_training_loop +from microbots.auto_memory.training.runner import run_training logger = getLogger(__name__) @@ -36,6 +36,45 @@ class LoopResult: final_outcome: EvalOutcome outcomes: list[EvalOutcome] = field(default_factory=list) +def run_training_loop( + repo_path: str, + feedback: str, + memory_dir: str, + model: str, + iterations: int = 1, +) -> None: + """Run ``run_training`` ``iterations`` times, reusing the same memory dir. + + Shared by the eval-loop's retrain step and any training-only entry + point (e.g. a CLI) that needs to run training without an eval task. + + Parameters + ---------- + repo_path : str + Absolute path to the repo to train against. + feedback : str + Feedback from a prior failed eval attempt, or ``""`` if none. + memory_dir : str + Directory where the training agent reads/writes memory files. + model : str + The model to use, in the format ``/``. + iterations : int + Number of training passes to run, each reusing the same + ``memory_dir``. Defaults to 1. + """ + for iteration in range(1, iterations + 1): + logger.info( + "run_training_loop: training iteration %d/%d", + iteration, + iterations, + ) + run_training( + repo_path=repo_path, + feedback=feedback, + memory_dir=memory_dir, + model=model, + ) + def run_train_eval_loop( repo_path: str, memory_dir: str, @@ -48,9 +87,10 @@ def run_train_eval_loop( Each round runs ``task.run(...)``. If the task passes, the loop returns immediately. If it fails, feedback is built from the round's - log and used to retrain via ``run_training_loop`` before the next - round. The round's log file is always deleted before the next round - starts. + log and used to retrain via ``run_training`` (called + ``training_iterations`` times, each pass reusing the same + ``memory_dir``) before the next round. The round's log file is + always deleted before the next round starts. Parameters ---------- @@ -102,7 +142,6 @@ def run_train_eval_loop( ) try: feedback = build_feedback(task, outcome, repo_path, model) - run_training_loop( repo_path=repo_path, feedback=feedback, diff --git a/src/microbots/auto_memory/task.py b/src/microbots/auto_memory/task.py index 8a1de46..9eb00ae 100644 --- a/src/microbots/auto_memory/task.py +++ b/src/microbots/auto_memory/task.py @@ -5,16 +5,8 @@ clean up afterward. """ -import tempfile from abc import ABC, abstractmethod from dataclasses import dataclass -from logging import getLogger -from pathlib import Path - -from microbots.bot.WritingBot import WritingBot -from microbots.tools.tool_definitions.memory_tool import MemoryTool - -logger = getLogger(__name__) @dataclass class CallbackResult: @@ -57,23 +49,31 @@ class EvalOutcome: class EvalTask(ABC): """Base class for a single evaluation task in the train <-> eval loop. - Subclasses must implement ``setup``, ``build_prompt``, and ``check``, - and may override ``teardown`` and ``run`` as needed. + Subclasses must implement ``run``. ``setup``, ``build_prompt``, + ``check``, and ``teardown`` are optional hooks subclasses may use + to structure their own ``run`` implementation (see + ``SweBenchVerifiedTask`` for an example), but nothing in this base + class calls them automatically. """ - @abstractmethod def setup(self, repo_path: str) -> None: - """Required. Prepare repo/environment before the agent runs. + """Optional. Prepare repo/environment before the agent runs. + + Not called automatically; only useful if your ``run`` + implementation calls it. Parameters ---------- repo_path : str Absolute path to the repo to prepare. """ + pass - @abstractmethod def build_prompt(self, repo_path: str) -> str: - """Required. Return the task prompt/instructions for the agent. + """Optional. Return the task prompt/instructions for the agent. + + Not called automatically; only useful if your ``run`` + implementation calls it. Parameters ---------- @@ -83,12 +83,16 @@ def build_prompt(self, repo_path: str) -> str: Returns ------- str - The prompt/instructions to give the agent. + The prompt/instructions to give the agent. Empty string by + default. """ + return "" - @abstractmethod def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackResult: - """Required. Verify whether the task was actually completed correctly. + """Optional. Verify whether the task was actually completed correctly. + + Not called automatically; only useful if your ``run`` + implementation calls it. Parameters ---------- @@ -103,8 +107,10 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes Returns ------- CallbackResult - The pass/fail verdict and its reason. + The pass/fail verdict and its reason. Passes by default. """ + return CallbackResult(passed=True, reason="not checked") + def teardown(self, repo_path: str) -> None: """Optional. Clean up anything setup() created. @@ -116,10 +122,9 @@ def teardown(self, repo_path: str) -> None: """ pass + @abstractmethod def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: - """Default eval iteration: setup -> build_prompt -> WritingBot -> check -> teardown. - Override this entirely if your task needs a different bot type, - additional tools, or custom retry/orchestration logic. + """Required. Run one eval iteration and return its outcome. Parameters ---------- @@ -137,53 +142,3 @@ def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: The result of this eval round, including the agent's output, the check verdict, and the round's log file path. """ - self.setup(repo_path) - log_path = tempfile.mktemp(suffix=".log") - Path(log_path).write_text("") - - try: - try: - prompt = self.build_prompt(repo_path) - bot = WritingBot( - model=model, - folder_to_mount=repo_path, - additional_tools=[MemoryTool(memory_dir=memory_dir)], - ) - bot_result = bot.run(prompt) - - with open(log_path, "a") as f: - f.write(f"Agent output:\n{bot_result.result}\n") - - if not bot_result.status: - reason = f"Bot run failed: {bot_result.error}" - with open(log_path, "a") as f: - f.write(f"\n{reason}\n") - result = CallbackResult(passed=False, reason=reason) - else: - result = self.check(repo_path, bot_result.result or "", log_path) - - return EvalOutcome( - passed=result.passed, - output=bot_result.result, - result=result, - log_path=log_path, - ) - except Exception as exc: - logger.exception( - "EvalTask.run: iteration raised %s", type(exc).__name__ - ) - with open(log_path, "a") as f: - f.write(f"\nException during eval iteration: {type(exc).__name__}: {exc}\n") - return EvalOutcome( - passed=False, - output=None, - result=CallbackResult( - passed=False, reason=f"{type(exc).__name__}: {exc}" - ), - log_path=log_path, - ) - finally: - try: - self.teardown(repo_path) - except Exception: - logger.exception("EvalTask.run: teardown() raised exception; ignoring") diff --git a/src/microbots/auto_memory/task_registry.py b/src/microbots/auto_memory/task_registry.py new file mode 100644 index 0000000..63367f7 --- /dev/null +++ b/src/microbots/auto_memory/task_registry.py @@ -0,0 +1,92 @@ +"""Registry for constructing ``EvalTask`` instances by name. + +Tasks self-register via the ``@register_task`` decorator, so new task +types can be added without editing a central if/elif factory function. +Callers (e.g. a CLI) look tasks up by name via ``create_task``. +""" + +import importlib +import pkgutil + +from microbots.auto_memory.task import EvalTask + +TASK_REGISTRY: dict[str, type[EvalTask]] = {} + +def register_task(name: str): + """Register an ``EvalTask`` subclass under ``name`` as a class decorator. + + Parameters + ---------- + name : str + The key other code will use to look up this task via + ``create_task``, e.g. ``"swebenchverified"``. + + Returns + ------- + Callable[[type[EvalTask]], type[EvalTask]] + A decorator that registers the class in ``TASK_REGISTRY`` and + returns it unchanged. + """ + + def decorator(task_cls: type[EvalTask]) -> type[EvalTask]: + """Register ``task_cls`` in ``TASK_REGISTRY`` under the enclosing ``name``. + + Parameters + ---------- + task_cls : type[EvalTask] + The ``EvalTask`` subclass to register. + + Returns + ------- + type[EvalTask] + ``task_cls``, unchanged. + """ + TASK_REGISTRY[name] = task_cls + return task_cls + + return decorator + +def create_task(name: str, **kwargs) -> EvalTask: + """Construct a registered ``EvalTask`` by name. + + Parameters + ---------- + name : str + The registered task name, e.g. ``"swebenchverified"``. + **kwargs + Keyword arguments forwarded to the task's constructor. + + Returns + ------- + EvalTask + The constructed task instance. + + Raises + ------ + ValueError + If ``name`` has not been registered via ``register_task``. + """ + try: + task_cls = TASK_REGISTRY[name] + except KeyError: + raise ValueError( + f"Unknown task {name!r}. Registered tasks: {sorted(TASK_REGISTRY)}" + ) from None + return task_cls(**kwargs) + +def discover_tasks(package_name: str = "microbots.auto_memory.eval") -> None: + """Import every module in ``package_name`` so ``@register_task`` fires. + + Adding a new task only requires dropping a new module into this + package (with its own ``@register_task`` decorator) — no other code + needs to change to make it discoverable. + + Parameters + ---------- + package_name : str + Dotted path of the package to scan for task modules. Defaults + to ``"microbots.auto_memory.eval"``. + """ + package = importlib.import_module(package_name) + for module_info in pkgutil.iter_modules(package.__path__): + importlib.import_module(f"{package_name}.{module_info.name}") diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py new file mode 100644 index 0000000..8754211 --- /dev/null +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -0,0 +1,477 @@ +"""Unit tests for microbots.auto_memory.eval.swebenchverified.""" + +import json +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/"))) + +from microbots.auto_memory.eval.swebenchverified import ( + SweBenchInstance, + SweBenchVerifiedTask, + load_instance_using_id, + load_instances_of_repo, +) +from microbots.auto_memory.task import CallbackResult + +MODULE = "microbots.auto_memory.eval.swebenchverified" + + +def _fake_rows(): + return [ + { + "instance_id": "django__django-1", + "repo": "django/django", + "base_commit": "abc123", + "problem_statement": "fix bug 1", + }, + { + "instance_id": "astropy__astropy-1", + "repo": "astropy/astropy", + "base_commit": "def456", + "problem_statement": "fix bug 2", + }, + { + "instance_id": "django__django-2", + "repo": "django/django", + "base_commit": "ghi789", + "problem_statement": "fix bug 3", + }, + ] + + +# --------------------------------------------------------------------------- +# load_instances_of_repo / load_instance_using_id +# --------------------------------------------------------------------------- + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instances_of_repo_filters_by_repo(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + instances = load_instances_of_repo(repo="django/django") + + assert [i.instance_id for i in instances] == ["django__django-1", "django__django-2"] + assert all(isinstance(i, SweBenchInstance) for i in instances) + + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instances_of_repo_returns_all_when_repo_none(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + instances = load_instances_of_repo(repo=None) + + assert len(instances) == 3 + + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instance_using_id_returns_matching_instance(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + instance = load_instance_using_id("astropy__astropy-1") + + assert instance.repo == "astropy/astropy" + assert instance.problem_statement == "fix bug 2" + + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instance_using_id_raises_when_not_found(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + with pytest.raises(ValueError, match="not found"): + load_instance_using_id("does-not-exist") + + +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.setup / build_prompt / teardown +# --------------------------------------------------------------------------- + +def _instance(): + return SweBenchInstance( + instance_id="django__django-1", + repo="django/django", + base_commit="abc123", + problem_statement="fix the bug", + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_setup_clones_and_checks_out_base_commit(mock_run): + task = SweBenchVerifiedTask(_instance()) + task.setup("/repo") + + clone_call, checkout_call = mock_run.call_args_list + assert clone_call.args[0] == ["git", "clone", "https://github.com/django/django.git", "/repo"] + assert checkout_call.args[0] == ["git", "checkout", "abc123"] + assert checkout_call.kwargs["cwd"] == "/repo" + + +@pytest.mark.unit +def test_build_prompt_returns_problem_statement(): + task = SweBenchVerifiedTask(_instance()) + assert task.build_prompt("/repo") == "fix the bug" + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_teardown_removes_repo_path(mock_run): + task = SweBenchVerifiedTask(_instance()) + task.teardown("/repo") + + mock_run.assert_called_once_with(["rm", "-rf", "/repo"], check=False) + + +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.check +# --------------------------------------------------------------------------- + +def _make_fake_subprocess_run(resolved: bool, raise_on_harness: bool = False): + """Build a subprocess.run stand-in that fakes git diff + the harness call.""" + + def _fake_run(cmd, **kwargs): + if cmd[:2] == ["git", "diff"]: + return MagicMock(stdout="diff --git a/x.py b/x.py\n+fix", stderr="", returncode=0) + if "swebench.harness.run_evaluation" in cmd: + if raise_on_harness: + raise RuntimeError("harness crashed") + run_id = cmd[cmd.index("--run_id") + 1] + report_dir = kwargs["cwd"] + instance_id = cmd[cmd.index("--instance_ids") + 1] + report = {"resolved_ids": [instance_id] if resolved else []} + (Path(report_dir) / f"microbots-eval-agent.{run_id}.json").write_text(json.dumps(report)) + return MagicMock(stdout="harness ran\n", stderr="", returncode=0) + return MagicMock(stdout="", stderr="", returncode=0) + + return _fake_run + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_passed_true_when_instance_in_resolved_ids(mock_run, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + result = task.check("/repo", "agent output", str(log_path)) + + assert result.passed is True + assert result.reason == "resolved" + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_passed_false_when_instance_not_in_resolved_ids(mock_run, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=False) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + result = task.check("/repo", "agent output", str(log_path)) + + assert result.passed is False + assert result.reason == "not resolved" + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_passed_false_when_report_file_never_written(mock_run, tmp_path): + # harness call succeeds but never writes a report file (e.g. it errored internally) + def _fake_run(cmd, **kwargs): + if cmd[:2] == ["git", "diff"]: + return MagicMock(stdout="diff", stderr="", returncode=0) + return MagicMock(stdout="", stderr="", returncode=1) + + mock_run.side_effect = _fake_run + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + result = task.check("/repo", "agent output", str(log_path)) + + assert result.passed is False + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_appends_to_log_file_without_truncating_existing_content(mock_run, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True) + log_path = tmp_path / "check.log" + log_path.write_text("Agent output:\nprevious content\n") + + task = SweBenchVerifiedTask(_instance()) + task.check("/repo", "agent output", str(log_path)) + + content = log_path.read_text() + assert "previous content" in content + assert "harness ran" in content + + +@pytest.mark.unit +@patch(f"{MODULE}.shutil.rmtree") +@patch(f"{MODULE}.subprocess.run") +def test_check_cleans_up_pred_path_and_report_dir_on_success(mock_run, mock_rmtree, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + task.check("/repo", "agent output", str(log_path)) + + mock_rmtree.assert_called_once() + + +@pytest.mark.unit +@patch(f"{MODULE}.shutil.rmtree") +@patch(f"{MODULE}.subprocess.run") +def test_check_cleans_up_even_when_harness_raises(mock_run, mock_rmtree, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True, raise_on_harness=True) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + with pytest.raises(RuntimeError, match="harness crashed"): + task.check("/repo", "agent output", str(log_path)) + + mock_rmtree.assert_called_once() + + +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.run +# --------------------------------------------------------------------------- + +@pytest.mark.unit +@patch(f"{MODULE}.MemoryTool") +@patch(f"{MODULE}.WritingBot") +def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock_memory_tool): + from microbots.MicroBot import BotRunResult + + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="agent did stuff", error=None) + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + calls = [] + task.setup = lambda repo_path: calls.append(("setup", repo_path)) + task.build_prompt = lambda repo_path: "do the task" + task.check = lambda repo_path, agent_output, log_path: ( + calls.append(("check", repo_path, agent_output, log_path)) + or CallbackResult(passed=True, reason="ok") + ) + task.teardown = lambda repo_path: calls.append(("teardown", repo_path)) + + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert calls[0] == ("setup", "/repo") + assert calls[1] == ("check", "/repo", "agent did stuff", outcome.log_path) + assert calls[2] == ("teardown", "/repo") + assert outcome.passed is True + assert outcome.output == "agent did stuff" + + +@pytest.mark.unit +@patch(f"{MODULE}.MemoryTool") +@patch(f"{MODULE}.WritingBot") +def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_tool): + from microbots.MicroBot import BotRunResult + + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + task.setup = lambda repo_path: None + task.build_prompt = lambda repo_path: "do the task" + task.teardown = lambda repo_path: None + seen_log_exists = {} + + def _check(repo_path, agent_output, log_path): + seen_log_exists["exists"] = os.path.exists(log_path) + return CallbackResult(passed=True, reason="ok") + + task.check = _check + + task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert seen_log_exists["exists"] is True + + +@pytest.mark.unit +@patch(f"{MODULE}.MemoryTool") +@patch(f"{MODULE}.WritingBot") +def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool): + from microbots.MicroBot import BotRunResult + + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed") + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + check_calls = [] + task.setup = lambda repo_path: None + task.build_prompt = lambda repo_path: "do the task" + task.check = lambda *a: check_calls.append(a) + task.teardown = lambda repo_path: None + + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert check_calls == [] + assert outcome.passed is False + assert "bot crashed" in outcome.result.reason + + +@pytest.mark.unit +@patch(f"{MODULE}.MemoryTool") +@patch(f"{MODULE}.WritingBot") +def test_run_converts_build_prompt_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): + task = SweBenchVerifiedTask(_instance()) + task.setup = lambda repo_path: None + task.teardown = lambda repo_path: None + + def _build_prompt(repo_path): + raise ValueError("bad prompt") + + task.build_prompt = _build_prompt + + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert outcome.passed is False + assert "bad prompt" in outcome.result.reason + with open(outcome.log_path) as f: + assert "bad prompt" in f.read() + + +@pytest.mark.unit +@patch(f"{MODULE}.MemoryTool") +@patch(f"{MODULE}.WritingBot") +def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): + from microbots.MicroBot import BotRunResult + + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + task.setup = lambda repo_path: None + task.build_prompt = lambda repo_path: "do the task" + task.teardown = lambda repo_path: None + + def _check(repo_path, agent_output, log_path): + raise RuntimeError("check exploded") + + task.check = _check + + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert outcome.passed is False + assert "check exploded" in outcome.result.reason + + +@pytest.mark.unit +@patch(f"{MODULE}.MemoryTool") +@patch(f"{MODULE}.WritingBot") +def test_run_still_calls_teardown_when_body_raises(mock_bot_cls, mock_memory_tool): + mock_bot_cls.side_effect = RuntimeError("bot construction failed") + + task = SweBenchVerifiedTask(_instance()) + teardown_calls = [] + task.setup = lambda repo_path: None + task.build_prompt = lambda repo_path: "do the task" + task.teardown = lambda repo_path: teardown_calls.append(repo_path) + + task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert teardown_calls == ["/repo"] + + +@pytest.mark.unit +@patch(f"{MODULE}.MemoryTool") +@patch(f"{MODULE}.WritingBot") +def test_run_teardown_exception_does_not_clobber_returned_outcome(mock_bot_cls, mock_memory_tool): + from microbots.MicroBot import BotRunResult + + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + task.setup = lambda repo_path: None + task.build_prompt = lambda repo_path: "do the task" + task.check = lambda repo_path, agent_output, log_path: CallbackResult(passed=True, reason="ok") + + def _teardown(repo_path): + raise RuntimeError("teardown boom") + + task.teardown = _teardown + + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + # teardown() raised, but the already-computed EvalOutcome must still be returned + assert outcome.passed is True + + + +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.add_cli_args / from_cli_args +# --------------------------------------------------------------------------- + +@pytest.mark.unit +def test_add_cli_args_registers_instance_id_and_repo_flags(): + import argparse + + parser = argparse.ArgumentParser() + SweBenchVerifiedTask.add_cli_args(parser) + + args = parser.parse_args(["--instance-id", "django__django-1", "--swebench-repo", "django/django"]) + assert args.instance_id == "django__django-1" + assert args.swebench_repo == "django/django" + + +@pytest.mark.unit +@patch(f"{MODULE}.load_instance_using_id") +def test_from_cli_args_uses_instance_id_when_given(mock_load_instance_using_id): + mock_load_instance_using_id.return_value = _instance() + args = MagicMock(instance_id="django__django-1", swebench_repo=None) + + tasks = SweBenchVerifiedTask.from_cli_args(args) + + mock_load_instance_using_id.assert_called_once_with("django__django-1") + assert len(tasks) == 1 + assert isinstance(tasks[0], SweBenchVerifiedTask) + assert tasks[0].instance == _instance() + + +@pytest.mark.unit +@patch(f"{MODULE}.load_instances_of_repo") +def test_from_cli_args_falls_back_to_repo_filter_when_no_instance_id(mock_load_instances_of_repo): + mock_load_instances_of_repo.return_value = [_instance(), _instance()] + args = MagicMock(instance_id=None, swebench_repo="django/django") + + tasks = SweBenchVerifiedTask.from_cli_args(args) + + mock_load_instances_of_repo.assert_called_once_with(repo="django/django") + assert len(tasks) == 2 + assert all(isinstance(t, SweBenchVerifiedTask) for t in tasks) + + +@pytest.mark.unit +@patch(f"{MODULE}.load_instances_of_repo") +def test_from_cli_args_handles_missing_attrs_gracefully(mock_load_instances_of_repo): + """Namespace without instance_id/swebench_repo attrs at all (not just None).""" + mock_load_instances_of_repo.return_value = [_instance()] + + class _EmptyArgs: + pass + + tasks = SweBenchVerifiedTask.from_cli_args(_EmptyArgs()) + + mock_load_instances_of_repo.assert_called_once_with(repo=None) + assert len(tasks) == 1 diff --git a/test/auto_memory/eval_swebenchverified/test_eval.py b/test/auto_memory/eval_swebenchverified/test_eval.py deleted file mode 100644 index 4115e45..0000000 --- a/test/auto_memory/eval_swebenchverified/test_eval.py +++ /dev/null @@ -1,243 +0,0 @@ -"""Unit tests for microbots.auto_memory.eval_swebenchverified.eval.""" - -import json -import os -import sys -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/"))) - -from microbots.auto_memory.eval_swebenchverified.eval import ( - SweBenchInstance, - SweBenchVerifiedTask, - load_instance_using_id, - load_instances_of_repo, -) - -MODULE = "microbots.auto_memory.eval_swebenchverified.eval" - - -def _fake_rows(): - return [ - { - "instance_id": "django__django-1", - "repo": "django/django", - "base_commit": "abc123", - "problem_statement": "fix bug 1", - }, - { - "instance_id": "astropy__astropy-1", - "repo": "astropy/astropy", - "base_commit": "def456", - "problem_statement": "fix bug 2", - }, - { - "instance_id": "django__django-2", - "repo": "django/django", - "base_commit": "ghi789", - "problem_statement": "fix bug 3", - }, - ] - - -# --------------------------------------------------------------------------- -# load_instances_of_repo / load_instance_using_id -# --------------------------------------------------------------------------- - -@pytest.mark.unit -@patch(f"{MODULE}.load_dataset") -def test_load_instances_of_repo_filters_by_repo(mock_load_dataset): - mock_load_dataset.return_value = _fake_rows() - - instances = load_instances_of_repo(repo="django/django") - - assert [i.instance_id for i in instances] == ["django__django-1", "django__django-2"] - assert all(isinstance(i, SweBenchInstance) for i in instances) - - -@pytest.mark.unit -@patch(f"{MODULE}.load_dataset") -def test_load_instances_of_repo_returns_all_when_repo_none(mock_load_dataset): - mock_load_dataset.return_value = _fake_rows() - - instances = load_instances_of_repo(repo=None) - - assert len(instances) == 3 - - -@pytest.mark.unit -@patch(f"{MODULE}.load_dataset") -def test_load_instance_using_id_returns_matching_instance(mock_load_dataset): - mock_load_dataset.return_value = _fake_rows() - - instance = load_instance_using_id("astropy__astropy-1") - - assert instance.repo == "astropy/astropy" - assert instance.problem_statement == "fix bug 2" - - -@pytest.mark.unit -@patch(f"{MODULE}.load_dataset") -def test_load_instance_using_id_raises_when_not_found(mock_load_dataset): - mock_load_dataset.return_value = _fake_rows() - - with pytest.raises(ValueError, match="not found"): - load_instance_using_id("does-not-exist") - - -# --------------------------------------------------------------------------- -# SweBenchVerifiedTask.setup / build_prompt / teardown -# --------------------------------------------------------------------------- - -def _instance(): - return SweBenchInstance( - instance_id="django__django-1", - repo="django/django", - base_commit="abc123", - problem_statement="fix the bug", - ) - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_setup_clones_and_checks_out_base_commit(mock_run): - task = SweBenchVerifiedTask(_instance()) - task.setup("/repo") - - clone_call, checkout_call = mock_run.call_args_list - assert clone_call.args[0] == ["git", "clone", "https://github.com/django/django.git", "/repo"] - assert checkout_call.args[0] == ["git", "checkout", "abc123"] - assert checkout_call.kwargs["cwd"] == "/repo" - - -@pytest.mark.unit -def test_build_prompt_returns_problem_statement(): - task = SweBenchVerifiedTask(_instance()) - assert task.build_prompt("/repo") == "fix the bug" - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_teardown_removes_repo_path(mock_run): - task = SweBenchVerifiedTask(_instance()) - task.teardown("/repo") - - mock_run.assert_called_once_with(["rm", "-rf", "/repo"], check=False) - - -# --------------------------------------------------------------------------- -# SweBenchVerifiedTask.check -# --------------------------------------------------------------------------- - -def _make_fake_subprocess_run(resolved: bool, raise_on_harness: bool = False): - """Build a subprocess.run stand-in that fakes git diff + the harness call.""" - - def _fake_run(cmd, **kwargs): - if cmd[:2] == ["git", "diff"]: - return MagicMock(stdout="diff --git a/x.py b/x.py\n+fix", stderr="", returncode=0) - if "swebench.harness.run_evaluation" in cmd: - if raise_on_harness: - raise RuntimeError("harness crashed") - run_id = cmd[cmd.index("--run_id") + 1] - report_dir = kwargs["cwd"] - instance_id = cmd[cmd.index("--instance_ids") + 1] - report = {"resolved_ids": [instance_id] if resolved else []} - (Path(report_dir) / f"microbots-eval-agent.{run_id}.json").write_text(json.dumps(report)) - return MagicMock(stdout="harness ran\n", stderr="", returncode=0) - return MagicMock(stdout="", stderr="", returncode=0) - - return _fake_run - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_check_passed_true_when_instance_in_resolved_ids(mock_run, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=True) - log_path = tmp_path / "check.log" - log_path.write_text("") - - task = SweBenchVerifiedTask(_instance()) - result = task.check("/repo", "agent output", str(log_path)) - - assert result.passed is True - assert result.reason == "resolved" - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_check_passed_false_when_instance_not_in_resolved_ids(mock_run, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=False) - log_path = tmp_path / "check.log" - log_path.write_text("") - - task = SweBenchVerifiedTask(_instance()) - result = task.check("/repo", "agent output", str(log_path)) - - assert result.passed is False - assert result.reason == "not resolved" - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_check_passed_false_when_report_file_never_written(mock_run, tmp_path): - # harness call succeeds but never writes a report file (e.g. it errored internally) - def _fake_run(cmd, **kwargs): - if cmd[:2] == ["git", "diff"]: - return MagicMock(stdout="diff", stderr="", returncode=0) - return MagicMock(stdout="", stderr="", returncode=1) - - mock_run.side_effect = _fake_run - log_path = tmp_path / "check.log" - log_path.write_text("") - - task = SweBenchVerifiedTask(_instance()) - result = task.check("/repo", "agent output", str(log_path)) - - assert result.passed is False - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_check_appends_to_log_file_without_truncating_existing_content(mock_run, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=True) - log_path = tmp_path / "check.log" - log_path.write_text("Agent output:\nprevious content\n") - - task = SweBenchVerifiedTask(_instance()) - task.check("/repo", "agent output", str(log_path)) - - content = log_path.read_text() - assert "previous content" in content - assert "harness ran" in content - - -@pytest.mark.unit -@patch(f"{MODULE}.shutil.rmtree") -@patch(f"{MODULE}.subprocess.run") -def test_check_cleans_up_pred_path_and_report_dir_on_success(mock_run, mock_rmtree, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=True) - log_path = tmp_path / "check.log" - log_path.write_text("") - - task = SweBenchVerifiedTask(_instance()) - task.check("/repo", "agent output", str(log_path)) - - mock_rmtree.assert_called_once() - - -@pytest.mark.unit -@patch(f"{MODULE}.shutil.rmtree") -@patch(f"{MODULE}.subprocess.run") -def test_check_cleans_up_even_when_harness_raises(mock_run, mock_rmtree, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=True, raise_on_harness=True) - log_path = tmp_path / "check.log" - log_path.write_text("") - - task = SweBenchVerifiedTask(_instance()) - with pytest.raises(RuntimeError, match="harness crashed"): - task.check("/repo", "agent output", str(log_path)) - - mock_rmtree.assert_called_once() diff --git a/test/auto_memory/test_cli.py b/test/auto_memory/test_cli.py new file mode 100644 index 0000000..da3ca8b --- /dev/null +++ b/test/auto_memory/test_cli.py @@ -0,0 +1,108 @@ +"""Unit tests for microbots.auto_memory.cli.""" + +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) + +from microbots.auto_memory.cli import main, parse_args + +MODULE = "microbots.auto_memory.cli" + +BASE_ARGS = ["--repo", "/repo", "--memory-dir", "/memory", "--model", "azure-openai/gpt-4o"] + + +@pytest.mark.unit +def test_parse_args_defaults(): + args = parse_args(BASE_ARGS) + + assert args.repo == "/repo" + assert args.memory_dir == "/memory" + assert args.model == "azure-openai/gpt-4o" + assert args.task is None + assert args.max_rounds == 5 + assert args.training_iterations == 1 + + +@pytest.mark.unit +def test_parse_args_with_known_task_adds_its_flags(): + args = parse_args(BASE_ARGS + ["--task", "swebenchverified", "--instance-id", "django__django-1"]) + + assert args.task == "swebenchverified" + assert args.instance_id == "django__django-1" + + +@pytest.mark.unit +def test_parse_args_rejects_unknown_task(): + with pytest.raises(SystemExit): + parse_args(BASE_ARGS + ["--task", "does-not-exist"]) + + +@pytest.mark.unit +@patch(f"{MODULE}.run_training_loop") +def test_main_runs_training_only_when_task_omitted(mock_run_training_loop): + main(BASE_ARGS) + + mock_run_training_loop.assert_called_once_with( + repo_path="/repo", + feedback="", + memory_dir="/memory", + model="azure-openai/gpt-4o", + iterations=1, + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.run_train_eval_loop") +@patch(f"{MODULE}.run_training_loop") +def test_main_does_not_run_eval_loop_when_task_omitted(mock_run_training_loop, mock_run_train_eval_loop): + main(BASE_ARGS) + + mock_run_train_eval_loop.assert_not_called() + + +@pytest.mark.unit +@patch(f"{MODULE}.run_train_eval_loop") +def test_main_runs_eval_loop_for_each_task_when_task_given(mock_run_train_eval_loop): + fake_task = MagicMock() + mock_run_train_eval_loop.return_value = MagicMock(passed=True, rounds_run=1) + + with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_cli_args=lambda args: [fake_task])}): + main(BASE_ARGS + ["--task", "swebenchverified"]) + + mock_run_train_eval_loop.assert_called_once_with( + repo_path="/repo", + memory_dir="/memory", + model="azure-openai/gpt-4o", + task=fake_task, + max_rounds=5, + training_iterations=1, + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.run_training_loop") +def test_main_does_not_run_training_only_path_when_task_given(mock_run_training_loop): + fake_task = MagicMock() + + with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_cli_args=lambda args: [fake_task])}): + with patch(f"{MODULE}.run_train_eval_loop") as mock_run_train_eval_loop: + mock_run_train_eval_loop.return_value = MagicMock(passed=True, rounds_run=1) + main(BASE_ARGS + ["--task", "swebenchverified"]) + + mock_run_training_loop.assert_not_called() + + +@pytest.mark.unit +@patch(f"{MODULE}.run_train_eval_loop") +def test_main_runs_eval_loop_once_per_returned_task(mock_run_train_eval_loop): + fake_tasks = [MagicMock(), MagicMock()] + mock_run_train_eval_loop.return_value = MagicMock(passed=False, rounds_run=5) + + with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_cli_args=lambda args: fake_tasks)}): + main(BASE_ARGS + ["--task", "swebenchverified"]) + + assert mock_run_train_eval_loop.call_count == 2 diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py index ac21773..e869c55 100644 --- a/test/auto_memory/test_orchestrator.py +++ b/test/auto_memory/test_orchestrator.py @@ -9,7 +9,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) -from microbots.auto_memory.orchestrator import LoopResult, run_train_eval_loop +from microbots.auto_memory.orchestrator import LoopResult, run_train_eval_loop, run_training_loop from microbots.auto_memory.task import CallbackResult, EvalOutcome @@ -183,3 +183,37 @@ def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_ru assert result.passed is True assert result.rounds_run == 2 assert not Path(log1).exists() + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +def test_run_training_loop_calls_run_training_once_by_default(mock_run_training): + run_training_loop(repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o") + + mock_run_training.assert_called_once_with( + repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o" + ) + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +def test_run_training_loop_calls_run_training_n_times(mock_run_training): + run_training_loop( + repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=3 + ) + + assert mock_run_training.call_count == 3 + mock_run_training.assert_called_with( + repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o" + ) + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +def test_run_training_loop_reuses_same_memory_dir_each_pass(mock_run_training): + run_training_loop( + repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=4 + ) + + memory_dirs = {call.kwargs["memory_dir"] for call in mock_run_training.call_args_list} + assert memory_dirs == {"/memory"} diff --git a/test/auto_memory/test_task.py b/test/auto_memory/test_task.py index bfc2cb6..aee0f9d 100644 --- a/test/auto_memory/test_task.py +++ b/test/auto_memory/test_task.py @@ -2,191 +2,61 @@ import os import sys -from unittest.mock import MagicMock, patch import pytest sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) from microbots.auto_memory.task import CallbackResult, EvalOutcome, EvalTask -from microbots.MicroBot import BotRunResult -class _StubTask(EvalTask): - """A minimal concrete EvalTask used to exercise the base run() logic.""" +class _RunOnlyTask(EvalTask): + """A task that overrides only run(), never touching the optional hooks.""" - def __init__(self, check_result=None, check_side_effect=None, build_prompt_side_effect=None): - self.setup_calls = [] - self.teardown_calls = [] - self.check_calls = [] - self._check_result = check_result or CallbackResult(passed=True, reason="ok") - self._check_side_effect = check_side_effect - self._build_prompt_side_effect = build_prompt_side_effect - - def setup(self, repo_path): - self.setup_calls.append(repo_path) - - def build_prompt(self, repo_path): - if self._build_prompt_side_effect: - raise self._build_prompt_side_effect - return "do the task" - - def check(self, repo_path, agent_output, log_path): - self.check_calls.append((repo_path, agent_output, log_path)) - if self._check_side_effect: - raise self._check_side_effect - return self._check_result - - def teardown(self, repo_path): - self.teardown_calls.append(repo_path) - - -class _RaisingTeardownTask(_StubTask): - def teardown(self, repo_path): - super().teardown(repo_path) - raise RuntimeError("teardown boom") - - -class _DefaultTeardownTask(EvalTask): - """A task that relies on EvalTask's default no-op teardown.""" - - def setup(self, repo_path): - pass - - def build_prompt(self, repo_path): - return "do the task" - - def check(self, repo_path, agent_output, log_path): - return CallbackResult(passed=True, reason="ok") + def run(self, repo_path, memory_dir, model): + return EvalOutcome( + passed=True, + output="custom output", + result=None, + log_path="/dev/null", + ) @pytest.mark.unit -def test_setup_and_check_are_abstract(): +def test_run_is_abstract(): with pytest.raises(TypeError): EvalTask() @pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock_memory_tool): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="agent did stuff", error=None) - mock_bot_cls.return_value = mock_bot - - task = _StubTask() +def test_subclass_overriding_only_run_is_instantiable(): + task = _RunOnlyTask() outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") - assert task.setup_calls == ["/repo"] - assert task.check_calls == [("/repo", "agent did stuff", outcome.log_path)] - assert task.teardown_calls == ["/repo"] assert outcome.passed is True - assert outcome.output == "agent did stuff" - - -@pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_tool): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) - mock_bot_cls.return_value = mock_bot - - seen_log_exists = {} - - class _CheckingTask(_StubTask): - def check(self, repo_path, agent_output, log_path): - seen_log_exists["exists"] = os.path.exists(log_path) - return super().check(repo_path, agent_output, log_path) - - task = _CheckingTask() - task.run("/repo", "/memory", "azure-openai/gpt-4o") - - assert seen_log_exists["exists"] is True - - -@pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed") - mock_bot_cls.return_value = mock_bot - - task = _StubTask() - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") - - assert task.check_calls == [] - assert outcome.passed is False - assert "bot crashed" in outcome.result.reason + assert outcome.output == "custom output" @pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_converts_build_prompt_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): - task = _StubTask(build_prompt_side_effect=ValueError("bad prompt")) - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") - - assert outcome.passed is False - assert "bad prompt" in outcome.result.reason - with open(outcome.log_path) as f: - assert "bad prompt" in f.read() +def test_default_setup_is_a_noop(): + # Should not raise. + _RunOnlyTask().setup("/repo") @pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) - mock_bot_cls.return_value = mock_bot - - task = _StubTask(check_side_effect=RuntimeError("check exploded")) - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") - - assert outcome.passed is False - assert "check exploded" in outcome.result.reason +def test_default_teardown_is_a_noop(): + # Should not raise. + _RunOnlyTask().teardown("/repo") @pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_still_calls_teardown_when_body_raises(mock_bot_cls, mock_memory_tool): - mock_bot_cls.side_effect = RuntimeError("bot construction failed") - - task = _StubTask() - task.run("/repo", "/memory", "azure-openai/gpt-4o") - - assert task.teardown_calls == ["/repo"] +def test_default_build_prompt_returns_empty_string(): + assert _RunOnlyTask().build_prompt("/repo") == "" @pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_teardown_exception_does_not_clobber_returned_outcome(mock_bot_cls, mock_memory_tool): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) - mock_bot_cls.return_value = mock_bot - - task = _RaisingTeardownTask() - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") - - # teardown() raised, but the already-computed EvalOutcome must still be returned - assert isinstance(outcome, EvalOutcome) - assert outcome.passed is True - - -@pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_uses_default_noop_teardown_when_not_overridden(mock_bot_cls, mock_memory_tool): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) - mock_bot_cls.return_value = mock_bot - - task = _DefaultTeardownTask() - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") - - assert outcome.passed is True +def test_default_check_passes_by_default(): + result = _RunOnlyTask().check("/repo", "output", "/log") + assert isinstance(result, CallbackResult) + assert result.passed is True diff --git a/test/auto_memory/test_task_registry.py b/test/auto_memory/test_task_registry.py new file mode 100644 index 0000000..d06247f --- /dev/null +++ b/test/auto_memory/test_task_registry.py @@ -0,0 +1,104 @@ +"""Unit tests for microbots.auto_memory.task_registry.""" + +import os +import sys +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) + +from microbots.auto_memory.task import EvalTask +from microbots.auto_memory.task_registry import TASK_REGISTRY, create_task, discover_tasks, register_task + +MODULE_PATH = "microbots.auto_memory.task_registry" + + +class _DummyTask(EvalTask): + def __init__(self, value=None): + self.value = value + + def setup(self, repo_path): + pass + + def build_prompt(self): + return "prompt" + + def check(self, output): + pass + + def teardown(self, repo_path): + pass + + def run(self, repo_path, memory_dir, model): + return super().run(repo_path, memory_dir, model) + + +@pytest.fixture(autouse=True) +def _clean_registry(): + """Snapshot/restore TASK_REGISTRY so tests don't leak state into each other.""" + original = dict(TASK_REGISTRY) + yield + TASK_REGISTRY.clear() + TASK_REGISTRY.update(original) + + +@pytest.mark.unit +def test_register_task_adds_class_to_registry(): + register_task("dummy")(_DummyTask) + + assert TASK_REGISTRY["dummy"] is _DummyTask + + +@pytest.mark.unit +def test_register_task_returns_class_unchanged(): + decorated = register_task("dummy")(_DummyTask) + + assert decorated is _DummyTask + + +@pytest.mark.unit +def test_create_task_constructs_registered_task_with_kwargs(): + register_task("dummy")(_DummyTask) + + task = create_task("dummy", value=42) + + assert isinstance(task, _DummyTask) + assert task.value == 42 + + +@pytest.mark.unit +def test_create_task_raises_for_unknown_name(): + with pytest.raises(ValueError, match="Unknown task 'nonexistent'"): + create_task("nonexistent") + + +@pytest.mark.unit +def test_discover_tasks_registers_swebenchverified(): + """Non-destructive: confirms discover_tasks() works against the real package.""" + discover_tasks() + + assert "swebenchverified" in TASK_REGISTRY + + +@pytest.mark.unit +@patch(f"{MODULE_PATH}.importlib.import_module") +@patch(f"{MODULE_PATH}.pkgutil.iter_modules") +def test_discover_tasks_imports_every_module_found_in_package(mock_iter_modules, mock_import_module): + fake_package = MagicMock() + fake_package.__path__ = ["/fake/path"] + mock_import_module.side_effect = ( + lambda name: fake_package if name == "fake.pkg" else MagicMock() + ) + mock_iter_modules.return_value = [ + SimpleNamespace(name="task_a"), + SimpleNamespace(name="task_b"), + ] + + discover_tasks(package_name="fake.pkg") + + mock_iter_modules.assert_called_once_with(["/fake/path"]) + mock_import_module.assert_any_call("fake.pkg") + mock_import_module.assert_any_call("fake.pkg.task_a") + mock_import_module.assert_any_call("fake.pkg.task_b") From 4a952abbb917f70b422502f31d2520c3cd68dd7e Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 2 Sep 2026 07:18:52 +0000 Subject: [PATCH 07/21] Refactor evalTask module and improve functionality --- pyproject.toml | 1 + requirements.txt | 11 - src/microbots/auto_memory/__init__.py | 2 +- src/microbots/auto_memory/analyzer.py | 2 +- src/microbots/auto_memory/cli.py | 64 +-- .../auto_memory/eval/swebenchverified.py | 128 ++++-- .../auto_memory/{task.py => evalTask.py} | 48 ++- src/microbots/auto_memory/orchestrator.py | 195 +++++++-- src/microbots/auto_memory/task_registry.py | 3 +- src/microbots/auto_memory/workdir.py | 383 ++++++++++++++++++ .../auto_memory/eval/test_swebenchverified.py | 94 +++-- test/auto_memory/test_analyzer.py | 2 +- test/auto_memory/test_cli.py | 127 ++++-- test/auto_memory/test_orchestrator.py | 295 ++++++++++++-- test/auto_memory/test_task.py | 27 +- test/auto_memory/test_task_registry.py | 2 +- test/auto_memory/test_workdir.py | 94 +++++ 17 files changed, 1276 insertions(+), 202 deletions(-) rename src/microbots/auto_memory/{task.py => evalTask.py} (73%) create mode 100644 src/microbots/auto_memory/workdir.py create mode 100644 test/auto_memory/test_workdir.py diff --git a/pyproject.toml b/pyproject.toml index 3917589..a66a138 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ requires-python = ">=3.11" ghcp = ["github-copilot-sdk==0.3.0"] azure_ad = ["azure-identity>=1.15.0"] dev = ["pre-commit>=3.7", "numpydoc>=1.8"] +training = ["datasets==4.5.0", "swebench==4.1.0"] [tool.setuptools.dynamic] dependencies = { file = ["requirements.txt"] } diff --git a/requirements.txt b/requirements.txt index 4bf03b5..bfb90ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,36 +10,26 @@ certifi==2025.8.3 charset-normalizer==3.4.3 click==8.3.0 coverage==7.11.3 -datasets==4.5.0 -dill==0.4.0 distro==1.9.0 docker==7.1.0 docstring_parser==0.17.0 fastapi==0.116.1 -filelock==3.20.3 frozenlist==1.7.0 -fsspec==2025.10.0 h11==0.16.0 -hf-xet==1.2.0 httpcore==1.0.9 httpx==0.28.1 -huggingface_hub==1.3.2 idna==3.10 iniconfig==2.1.0 jiter==0.11.0 markdown-it-py==4.0.0 mdurl==0.1.2 multidict==6.6.4 -multiprocess==0.70.18 -numpy==1.26.4 openai==1.107.3 packaging==25.0 -pandas==3.0.0 pexpect==4.9.0 pluggy==1.6.0 propcache==0.3.2 ptyprocess==0.7.0 -pyarrow==23.0.0 pydantic==2.11.9 pydantic_core==2.33.2 Pygments==2.19.2 @@ -62,5 +52,4 @@ typing-inspection==0.4.1 typing_extensions==4.15.0 urllib3==2.5.0 uvicorn==0.35.0 -xxhash==3.6.0 yarl==1.20.1 diff --git a/src/microbots/auto_memory/__init__.py b/src/microbots/auto_memory/__init__.py index 27f9a4b..e959bfa 100644 --- a/src/microbots/auto_memory/__init__.py +++ b/src/microbots/auto_memory/__init__.py @@ -4,5 +4,5 @@ an evaluation task and run it in a loop against a training agent. """ -from .task import CallbackResult, EvalOutcome, EvalTask +from .evalTask import CallbackResult, EvalOutcome, EvalTask from .orchestrator import LoopResult, run_train_eval_loop \ No newline at end of file diff --git a/src/microbots/auto_memory/analyzer.py b/src/microbots/auto_memory/analyzer.py index 83e75cf..ca35c16 100644 --- a/src/microbots/auto_memory/analyzer.py +++ b/src/microbots/auto_memory/analyzer.py @@ -7,7 +7,7 @@ from logging import getLogger -from microbots.auto_memory.task import EvalOutcome, EvalTask +from microbots.auto_memory.evalTask import EvalOutcome, EvalTask from microbots.bot.LogAnalysisBot import LogAnalysisBot from microbots.MicroBot import BotRunResult diff --git a/src/microbots/auto_memory/cli.py b/src/microbots/auto_memory/cli.py index 98d3147..66ab0e6 100644 --- a/src/microbots/auto_memory/cli.py +++ b/src/microbots/auto_memory/cli.py @@ -3,16 +3,19 @@ Two modes, selected by ``--task``: - ``--task `` given: run the full train <-> eval loop for that - task (via ``run_train_eval_loop``). -- ``--task`` omitted: train only, no eval task (via ``run_training_loop``, - with empty feedback). + task. +- ``--task`` omitted: train only, no eval task, with empty feedback. + +Both modes are dispatched via ``orchestrator.run``. """ import argparse import logging +from pathlib import Path -from microbots.auto_memory.orchestrator import run_train_eval_loop, run_training_loop +from microbots.auto_memory.orchestrator import run from microbots.auto_memory.task_registry import TASK_REGISTRY, discover_tasks +from microbots.auto_memory.workdir import load_config, require_workdir, resolve_workdir logger = logging.getLogger(__name__) @@ -20,7 +23,10 @@ discover_tasks() def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse CLI args, including task-specific args when ``--task`` is given. + """Parse the CLI's top-level args. + + Task-specific values (e.g. an eval task's instance ID) are not + parsed here; they come from the workdir's config file instead. Parameters ---------- @@ -33,22 +39,19 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: The parsed args. """ parser = argparse.ArgumentParser(description="Run the auto-memory train/eval loop.") - parser.add_argument("--repo", required=True, help="Absolute path to the repo.") - parser.add_argument("--memory-dir", required=True, help="Directory for memory files.") parser.add_argument("--model", required=True, help='Model, e.g. "azure-openai/gpt-5.5".') + parser.add_argument( + "--workdir", + help="Directory holding this run's files (repo clone, logs, memory, " + "config). Defaults to './workdir' relative to the current directory.", + ) parser.add_argument( "--task", choices=sorted(TASK_REGISTRY), help="Eval task to run. Omit to only run training, with no eval task.", ) parser.add_argument("--max-rounds", type=int, default=5) - parser.add_argument("--training-iterations", type=int, default=1) - - # First pass just to discover --task, so we can register its - # task-specific flags before the real parse. - known_args, _ = parser.parse_known_args(argv) - if known_args.task: - TASK_REGISTRY[known_args.task].add_cli_args(parser) + parser.add_argument("--training-iterations", type=int, default=10) return parser.parse_args(argv) @@ -62,29 +65,28 @@ def main(argv: list[str] | None = None) -> None: """ args = parse_args(argv) - if not args.task: - run_training_loop( - repo_path=args.repo, - feedback="", - memory_dir=args.memory_dir, - model=args.model, - iterations=args.training_iterations, - ) - return + workdir = Path(args.workdir) if args.workdir else resolve_workdir() + require_workdir(workdir) - task_cls = TASK_REGISTRY[args.task] - for task in task_cls.from_cli_args(args): - result = run_train_eval_loop( - repo_path=args.repo, - memory_dir=args.memory_dir, + config = load_config(workdir) + tasks = ( + TASK_REGISTRY[args.task].from_config(config.get("task_args", {})) + if args.task + else [None] + ) + for task in tasks: + result = run( + workdir=workdir, model=args.model, task=task, max_rounds=args.max_rounds, training_iterations=args.training_iterations, + config=config, ) - logger.info( - "task=%s passed=%s rounds_run=%d", args.task, result.passed, result.rounds_run - ) + if result is not None: + logger.info( + "task=%s passed=%s rounds_run=%d", args.task, result.passed, result.rounds_run + ) if __name__ == "__main__": main() diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index 2cd974b..96017b8 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -13,18 +13,44 @@ import tempfile import uuid from dataclasses import dataclass +from functools import lru_cache from logging import getLogger from pathlib import Path from datasets import load_dataset -from microbots.auto_memory.task import CallbackResult, EvalOutcome, EvalTask +from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome, EvalTask from microbots.auto_memory.task_registry import register_task from microbots.bot.WritingBot import WritingBot from microbots.tools.tool_definitions.memory_tool import MemoryTool logger = getLogger(__name__) -SWE_BENCH_SUITE = "SWE-bench/SWE-bench_Verified" +SWE_BENCH_VERIFIED = "SWE-bench/SWE-bench_Verified" +EVAL_AGENT_MODEL_NAME = "microbots-eval-agent" + + +@lru_cache(maxsize=None) +def _load_dataset_rows(dataset_name: str): + """Load and cache ``dataset_name``'s ``test`` split for the process's lifetime. + + ``load_dataset`` caches the downloaded files on disk, but still + re-reads and rebuilds the in-memory ``Dataset`` object on every + call. Since ``load_instances_of_repo``/``load_instance_using_id`` + may each be called many times (e.g. once per eval task instance), + this wraps ``load_dataset`` with an in-memory cache keyed by + ``dataset_name``, so the dataset is only loaded once per process. + + Parameters + ---------- + dataset_name : str + Hugging Face dataset name to load. + + Returns + ------- + datasets.Dataset + The loaded ``test`` split. + """ + return load_dataset(dataset_name, split="test") @dataclass @@ -50,7 +76,7 @@ class SweBenchInstance: def load_instances_of_repo( - dataset_name: str = SWE_BENCH_SUITE, + dataset_name: str = SWE_BENCH_VERIFIED, repo: str | None = None, ) -> list[SweBenchInstance]: """Load all dataset instances, optionally filtered to a single repo. @@ -59,7 +85,7 @@ def load_instances_of_repo( ---------- dataset_name : str Hugging Face dataset name to load. Defaults to - ``SWE_BENCH_SUITE``. + ``SWE_BENCH_VERIFIED``. repo : str | None If given, only instances whose ``repo`` matches this value are returned, e.g. ``"django/django"``. If ``None``, all instances @@ -70,7 +96,7 @@ def load_instances_of_repo( list[SweBenchInstance] The matching instances. """ - rows = load_dataset(dataset_name, split="test") + rows = _load_dataset_rows(dataset_name) instances = [ SweBenchInstance( instance_id=row["instance_id"], @@ -83,7 +109,7 @@ def load_instances_of_repo( ] return instances -def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_SUITE) -> SweBenchInstance: +def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIFIED) -> SweBenchInstance: """Load a single dataset instance by its instance ID. Parameters @@ -92,7 +118,7 @@ def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_SUITE The instance ID to look up, e.g. ``"django__django-11099"``. dataset_name : str Hugging Face dataset name to load. Defaults to - ``SWE_BENCH_SUITE``. + ``SWE_BENCH_VERIFIED``. Returns ------- @@ -105,7 +131,7 @@ def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_SUITE If no instance with the given ``instance_id`` exists in the dataset. """ - rows = load_dataset(dataset_name, split="test") + rows = _load_dataset_rows(dataset_name) for row in rows: if row["instance_id"] == instance_id: return SweBenchInstance( @@ -130,13 +156,15 @@ class SweBenchVerifiedTask(EvalTask): The dataset instance this task evaluates against. """ - def __init__(self, instance: SweBenchInstance): - """Initialize the task for a single dataset instance. + def __init__(self, instance: SweBenchInstance | None = None): + """Initialize the task, optionally for a single dataset instance. Parameters ---------- - instance : SweBenchInstance - The dataset instance this task evaluates against. + instance : SweBenchInstance | None + The dataset instance this task evaluates against. May be + omitted and set later via ``self.instance``, but must be + set before any other method on this task is called. """ self.instance = instance @@ -181,6 +209,62 @@ def from_cli_args(cls, args: argparse.Namespace) -> list["SweBenchVerifiedTask"] instances = load_instances_of_repo(repo=getattr(args, "swebench_repo", None)) return [cls(instance) for instance in instances] + @classmethod + def from_config(cls, task_args: dict) -> list["SweBenchVerifiedTask"]: + """Build task(s) from a config's ``task_args`` dict. + + Parameters + ---------- + task_args : dict + Task-specific config values, expected to include + ``instance_id`` and/or ``swebench_repo`` (mirrors + ``add_cli_args``'s flags). + + Returns + ------- + list[SweBenchVerifiedTask] + One task per matching dataset instance. A single-element + list when ``instance_id`` is given. + """ + if task_args.get("instance_id"): + instances = [load_instance_using_id(task_args["instance_id"])] + else: + instances = load_instances_of_repo(repo=task_args.get("swebench_repo")) + return [cls(instance) for instance in instances] + + @property + def task_id(self) -> str: + """Return this instance's SWE-bench-verified ``instance_id``. + + Returns + ------- + str + The dataset instance's ``instance_id``. + """ + return self.instance.instance_id + + def build_result(self, outcome: EvalOutcome) -> dict: + """Summarize a round's outcome, including the instance's dataset fields. + + Parameters + ---------- + outcome : EvalOutcome + The round's outcome to summarize. + + Returns + ------- + dict + ``passed``/``reason`` plus ``instance_id``, ``repo``, and + ``base_commit`` identifying which dataset row this is. + """ + return { + "passed": outcome.result.passed, + "reason": outcome.result.reason, + "instance_id": self.instance.instance_id, + "repo": self.instance.repo, + "base_commit": self.instance.base_commit, + } + def setup(self, repo_path: str) -> None: """Clone the instance's repo and check out its base commit. @@ -197,14 +281,9 @@ def setup(self, repo_path: str) -> None: ["git", "checkout", self.instance.base_commit], cwd=repo_path, check=True ) - def build_prompt(self, repo_path: str) -> str: + def build_prompt(self) -> str: """Return the instance's issue text as the agent's prompt. - Parameters - ---------- - repo_path : str - Absolute path to the repo the agent will operate on. - Returns ------- str @@ -240,7 +319,7 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes ).stdout run_id = f"microbots-{uuid.uuid4().hex[:8]}" - model_name_or_path = "microbots-eval-agent" + model_name_or_path = EVAL_AGENT_MODEL_NAME pred_path = Path(tempfile.mktemp(suffix=".json")) report_dir = Path(tempfile.mkdtemp()) pred_path.write_text(json.dumps([{ @@ -252,7 +331,7 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes try: proc = subprocess.run( [sys.executable, "-m", "swebench.harness.run_evaluation", - "--dataset_name", SWE_BENCH_SUITE, + "--dataset_name", SWE_BENCH_VERIFIED, "--max_workers", "1", "--predictions_path", str(pred_path), "--run_id", run_id, @@ -286,7 +365,7 @@ def teardown(self, repo_path: str) -> None: """ subprocess.run(["rm", "-rf", repo_path], check=False) - def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: + def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> EvalOutcome: """Run one eval iteration: setup -> build_prompt -> WritingBot -> check -> teardown. Parameters @@ -298,6 +377,9 @@ def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: ``MemoryTool``. model : str The model to use, in the format ``/``. + log_path : str + Path to write this round's log to. Caller-provided, so the + log persists under the run's own layout. Returns ------- @@ -306,12 +388,12 @@ def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: the check verdict, and the round's log file path. """ self.setup(repo_path) - log_path = tempfile.mktemp(suffix=".log") + Path(log_path).parent.mkdir(parents=True, exist_ok=True) Path(log_path).write_text("") try: try: - prompt = self.build_prompt(repo_path) + prompt = self.build_prompt() bot = WritingBot( model=model, folder_to_mount=repo_path, diff --git a/src/microbots/auto_memory/task.py b/src/microbots/auto_memory/evalTask.py similarity index 73% rename from src/microbots/auto_memory/task.py rename to src/microbots/auto_memory/evalTask.py index 9eb00ae..6a8925c 100644 --- a/src/microbots/auto_memory/task.py +++ b/src/microbots/auto_memory/evalTask.py @@ -56,6 +56,41 @@ class EvalTask(ABC): class calls them automatically. """ + @property + def task_id(self) -> str: + """Identifier for this task instance, used to name its output folder. + + Defaults to the class name, which is fine for tasks with only + one instance per run. Override for tasks with several distinct + instances per class (e.g. ``SweBenchVerifiedTask``, where each + dataset row needs its own folder). + + Returns + ------- + str + This task instance's identifier. + """ + return type(self).__name__ + + def build_result(self, outcome: EvalOutcome) -> dict: + """Optional. Build the dict written to this round's ``result.json``. + + Not called automatically; the orchestrator calls this after + each round to decide what to persist. Override to include + task-specific details (e.g. dataset fields, repo info). + + Parameters + ---------- + outcome : EvalOutcome + The round's outcome to summarize. + + Returns + ------- + dict + JSON-serializable summary. Defaults to ``passed``/``reason``. + """ + return {"passed": outcome.result.passed, "reason": outcome.result.reason} + def setup(self, repo_path: str) -> None: """Optional. Prepare repo/environment before the agent runs. @@ -69,17 +104,12 @@ def setup(self, repo_path: str) -> None: """ pass - def build_prompt(self, repo_path: str) -> str: + def build_prompt(self) -> str: """Optional. Return the task prompt/instructions for the agent. Not called automatically; only useful if your ``run`` implementation calls it. - Parameters - ---------- - repo_path : str - Absolute path to the repo the agent will operate on. - Returns ------- str @@ -123,7 +153,7 @@ def teardown(self, repo_path: str) -> None: pass @abstractmethod - def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: + def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> EvalOutcome: """Required. Run one eval iteration and return its outcome. Parameters @@ -135,6 +165,10 @@ def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: ``MemoryTool``. model : str The model to use, in the format ``/``. + log_path : str + Path to write this round's log to. Caller-provided (e.g. a + workdir-managed path) so logs persist under the run's + layout instead of each task inventing its own temp file. Returns ------- diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index f4ca198..0c29538 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -8,10 +8,20 @@ from dataclasses import dataclass, field from logging import getLogger from pathlib import Path +import json +import subprocess from microbots.auto_memory.analyzer import build_feedback -from microbots.auto_memory.task import EvalOutcome, EvalTask +from microbots.auto_memory.evalTask import EvalOutcome, EvalTask from microbots.auto_memory.training.runner import run_training +from microbots.auto_memory.workdir import ( + eval_log_path, + eval_result_path, + load_config, + load_round_memory, + repo_dir, + save_round_memory, +) logger = getLogger(__name__) @@ -36,12 +46,68 @@ class LoopResult: final_outcome: EvalOutcome outcomes: list[EvalOutcome] = field(default_factory=list) +def clone_repo(url: str, repo_path: Path) -> None: + """Clone ``url`` into ``repo_path`` if it isn't already cloned there. + + Parameters + ---------- + url : str + Git URL (or local path) to clone from. + repo_path : Path + Destination directory for the clone. If it already exists (e.g. + a previous round already cloned here), this is a no-op. + """ + if repo_path.exists(): + return + subprocess.run(["git", "clone", url, str(repo_path)], check=True) + +def reset_repo(repo_path: Path, base_commit: str) -> None: + """Reset ``repo_path`` to ``base_commit``, discarding all local changes. + + Runs ``git reset --hard `` followed by ``git clean -fd``, + so every round/instance starts from the same pristine state instead + of carrying forward whatever a previous round or eval attempt left + behind. + + Parameters + ---------- + repo_path : Path + Path to the repo to reset. + base_commit : str + Commit-ish to reset to. + """ + subprocess.run(["git", "reset", "--hard", base_commit], cwd=repo_path, check=True) + subprocess.run(["git", "clean", "-fd"], cwd=repo_path, check=True) + +def write_eval_result(workdir: Path, round_num: int, task: EvalTask, outcome: EvalOutcome) -> None: + """Write a round's eval result to ``result.json``. + + Delegates the content to ``task.build_result(outcome)`` so each + task decides what's worth persisting (e.g. ``SweBenchVerifiedTask`` + includes its dataset instance's fields). + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number this outcome belongs to. + task : EvalTask + The task that produced ``outcome``, used for both its + ``task_id`` (folder name) and ``build_result`` (file content). + outcome : EvalOutcome + The round's outcome to persist. + """ + path = eval_result_path(workdir, round_num, task.task_id) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(task.build_result(outcome), indent=2)) + def run_training_loop( repo_path: str, feedback: str, memory_dir: str, model: str, - iterations: int = 1, + iterations: int = 10, ) -> None: """Run ``run_training`` ``iterations`` times, reusing the same memory dir. @@ -60,7 +126,7 @@ def run_training_loop( The model to use, in the format ``/``. iterations : int Number of training passes to run, each reusing the same - ``memory_dir``. Defaults to 1. + ``memory_dir``. Defaults to 10. """ for iteration in range(1, iterations + 1): logger.info( @@ -77,27 +143,37 @@ def run_training_loop( def run_train_eval_loop( repo_path: str, - memory_dir: str, + workdir: Path, model: str, task: EvalTask, max_rounds: int = 5, - training_iterations: int = 1, + training_iterations: int = 10, ) -> LoopResult: """Run an eval task in a loop, retraining on failure until it passes. - Each round runs ``task.run(...)``. If the task passes, the loop - returns immediately. If it fails, feedback is built from the round's - log and used to retrain via ``run_training`` (called - ``training_iterations`` times, each pass reusing the same - ``memory_dir``) before the next round. The round's log file is - always deleted before the next round starts. + Each round loads the current top-level memory into its own + ``rounds_/round_N/memory`` (carried forward from the + previous round, or empty on round 1), then runs ``task.run(...)`` + against it, writing its log to a workdir-managed path + (``rounds_/round_N/eval/eval.log``) so it persists. Since + each eval task instance gets its own ``rounds_`` dir, + different instances sharing the same ``workdir`` never collide on + round numbers, and each instance's per-round memory is preserved + individually. If the task passes, the loop returns immediately. If + it fails, feedback is built from the round's log and used to + retrain via ``run_training`` (called ``training_iterations`` times, + each pass reusing the same round memory dir) before the next round. + Either way, the round's result is written to ``result.json`` and + its memory is saved back to the top-level memory dir before the + next round starts. Parameters ---------- repo_path : str Absolute path to the repo to evaluate and train against. - memory_dir : str - Directory where the training agent reads/writes memory files. + workdir : Path + This run's workdir, used to carry memory forward between rounds + (see ``microbots.auto_memory.workdir``). model : str The model to use, in the format ``/``. task : EvalTask @@ -106,7 +182,7 @@ def run_train_eval_loop( Maximum number of train/eval rounds to attempt. Defaults to 5. training_iterations : int Number of training passes to run per retraining round, each - reusing the same ``memory_dir``. Defaults to 1. + reusing the same round memory dir. Defaults to 10. Returns ------- @@ -116,28 +192,31 @@ def run_train_eval_loop( """ outcomes: list[EvalOutcome] = [] - for round_idx in range(max_rounds): + for round_idx in range(1, max_rounds+1): logger.info( - "run_train_eval_loop: round %d/%d starting", round_idx + 1, max_rounds + "run_train_eval_loop: round %d/%d starting", round_idx, max_rounds + ) + memory_dir = str(load_round_memory(workdir, round_idx, instance_id=task.task_id)) + outcome = task.run( + repo_path, memory_dir, model, str(eval_log_path(workdir, round_idx, task.task_id)) ) - outcome = task.run(repo_path, memory_dir, model) outcomes.append(outcome) try: if outcome.passed: logger.info( - "run_train_eval_loop: passed on round %d/%d", round_idx + 1, max_rounds + "run_train_eval_loop: passed on round %d/%d", round_idx, max_rounds ) return LoopResult( passed=True, - rounds_run=round_idx + 1, + rounds_run=round_idx, final_outcome=outcome, outcomes=outcomes, ) logger.info( "run_train_eval_loop: round %d failed (%s), retraining", - round_idx + 1, + round_idx, outcome.result.reason, ) try: @@ -153,10 +232,11 @@ def run_train_eval_loop( logger.exception( "run_train_eval_loop: round %d failed to build feedback/retrain; " "continuing to next round without retraining", - round_idx + 1, + round_idx, ) finally: - Path(outcome.log_path).unlink(missing_ok=True) + write_eval_result(workdir, round_idx, task, outcome) + save_round_memory(workdir, round_idx, instance_id=task.task_id) logger.info( "run_train_eval_loop: exhausted %d rounds without passing", max_rounds @@ -166,4 +246,73 @@ def run_train_eval_loop( rounds_run=max_rounds, final_outcome=outcomes[-1], outcomes=outcomes, - ) \ No newline at end of file + ) + +def run( + workdir: Path, + model: str, + task: EvalTask | None, + max_rounds: int = 5, + training_iterations: int = 10, + config: dict | None = None, +) -> LoopResult | None: + """Run training only, or the full train/eval loop, depending on ``task``. + + Parameters + ---------- + workdir : Path + This run's workdir (see ``microbots.auto_memory.workdir``), + holding ``config.yaml``, the shared repo clone, and all output. + model : str + The model to use, in the format ``/``. + task : EvalTask | None + The eval task to run each round, or ``None`` to only run + training (with empty feedback, once per ``training_iterations``). + max_rounds : int + Maximum number of train/eval rounds to attempt, if ``task`` is + given. Defaults to 5. + training_iterations : int + Number of training passes to run per retraining round, each + reusing the same round memory dir. Defaults to 10. + config : dict | None + This run's already-loaded ``config.yaml`` contents. If ``None`` + (the default), it is loaded from ``workdir`` here. Callers that + invoke ``run`` repeatedly for the same ``workdir`` (e.g. once + per eval task) can load it once and pass it in, to avoid + re-reading/re-parsing the file on every call. + + Returns + ------- + LoopResult | None + The eval loop's result if ``task`` was given, otherwise ``None``. + """ + if config is None: + config = load_config(workdir) + repo_url = config.get("repo") + if repo_url: + clone_repo(repo_url, repo_dir(workdir)) + + repo_path = str(repo_dir(workdir)) + + if task is None: + # Train-only mode has no rounds of its own; round 1 is just a + # scratch dir seeded from (and saved back to) top-level memory. + memory_dir = str(load_round_memory(workdir, 1)) + run_training_loop( + repo_path=repo_path, + feedback="", + memory_dir=memory_dir, + model=model, + iterations=training_iterations, + ) + save_round_memory(workdir, 1) + return None + + return run_train_eval_loop( + repo_path=repo_path, + workdir=workdir, + model=model, + task=task, + max_rounds=max_rounds, + training_iterations=training_iterations, + ) diff --git a/src/microbots/auto_memory/task_registry.py b/src/microbots/auto_memory/task_registry.py index 63367f7..63bfb74 100644 --- a/src/microbots/auto_memory/task_registry.py +++ b/src/microbots/auto_memory/task_registry.py @@ -8,7 +8,7 @@ import importlib import pkgutil -from microbots.auto_memory.task import EvalTask +from microbots.auto_memory.evalTask import EvalTask TASK_REGISTRY: dict[str, type[EvalTask]] = {} @@ -46,6 +46,7 @@ def decorator(task_cls: type[EvalTask]) -> type[EvalTask]: return decorator +# Not being used currently, but kept it for future use if required. def create_task(name: str, **kwargs) -> EvalTask: """Construct a registered ``EvalTask`` by name. diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py new file mode 100644 index 0000000..771066c --- /dev/null +++ b/src/microbots/auto_memory/workdir.py @@ -0,0 +1,383 @@ +"""Path/layout helpers for a training run's workdir. + +Centralizes every path this package reads or writes under a run's +``workdir`` (config, repo clone, logs, memory, and per-round/per-eval +outputs), so callers never hard-code layout details themselves. +""" + +from pathlib import Path +import shutil + +import yaml + +WORKDIR_NAME = "workdir" +CONFIG_FILENAME = "config.yaml" +REPO_DIRNAME = "repo" +RUN_LOG_FILENAME = "run.log" +MEMORY_DIRNAME = "memory" +ROUNDS_DIRNAME = "rounds" +ROUND_LOG_FILENAME = "round.log" +ROUND_PATCH_FILENAME = "repo.patch" +EVAL_DIRNAME = "eval" +RESULT_FILENAME = "result.json" +EVAL_LOG_FILENAME = "eval.log" + + +def resolve_workdir(base: Path | None = None) -> Path: + """Resolve the fixed workdir path relative to ``base``. + + Parameters + ---------- + base : Path | None + Directory to resolve ``workdir/`` relative to. Defaults to the + current working directory. + + Returns + ------- + Path + ``workdir`` resolved relative to ``base`` (or ``Path.cwd()``). + """ + return (base or Path.cwd()) / WORKDIR_NAME + + +def require_workdir(workdir: Path) -> None: + """Validate that ``workdir`` exist. + + Parameters + ---------- + workdir : Path + The workdir to validate. + + Raises + ------ + FileNotFoundError + If ``workdir`` does not exist. + """ + if not workdir.is_dir(): + raise FileNotFoundError(f"workdir not found: {workdir}") + + +def config_path(workdir: Path) -> Path: + """Return the path to ``workdir``'s config file. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/config.yaml``. + """ + return workdir / CONFIG_FILENAME + + +def load_config(workdir: Path) -> dict: + """Load and parse ``workdir``'s config file. + + Parameters + ---------- + workdir : Path + The workdir whose config file should be loaded. + + Returns + ------- + dict + The parsed config, or ``{}`` if the config file doesn't exist + or is empty. + """ + path = config_path(workdir) + if not path.is_file(): + return {} + return yaml.safe_load(path.read_text()) or {} + + +def repo_dir(workdir: Path) -> Path: + """Return the path to the single cloned repo shared across rounds. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/repo``. + """ + return workdir / REPO_DIRNAME + + +def run_log_path(workdir: Path) -> Path: + """Return the path to the top-level orchestrator log. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/run.log``. + """ + return workdir / RUN_LOG_FILENAME + + +def memory_dir(workdir: Path) -> Path: + """Return the path to the current top-level (latest) memory directory. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/memory``. + """ + return workdir / MEMORY_DIRNAME + + +def round_dir( + workdir: Path, round_num: int, *, instance_id: str | None = None, create: bool = False +) -> Path: + """Return (and optionally create) the directory for a training round. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number. + instance_id : str | None + If given, rounds are kept under a per-instance rounds dir + (``rounds_{instance_id}``) instead of the shared ``rounds`` dir, + so different eval task instances sharing the same ``workdir`` + don't collide on round numbers. Pass the eval task's + ``task_id`` when running an eval task; omit for training-only + mode. + create : bool + If True, create the directory (and parents) if missing. + + Returns + ------- + Path + ``workdir/rounds/round_{round_num}`` (no ``instance_id``), or + ``workdir/rounds_{instance_id}/round_{round_num}``. + """ + rounds_dirname = f"{ROUNDS_DIRNAME}_{instance_id}" if instance_id else ROUNDS_DIRNAME + path = workdir / rounds_dirname / f"round_{round_num}" + if create: + path.mkdir(parents=True, exist_ok=True) + return path + + +def round_memory_dir(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: + """Return the path to a round's own memory snapshot (a directory). + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number. + instance_id : str | None + The eval task's ``task_id``, if running an eval task (see + ``round_dir``). Omit for training-only mode. + + Returns + ------- + Path + This round's own memory directory. + """ + return round_dir(workdir, round_num, instance_id=instance_id) / MEMORY_DIRNAME + + +def load_round_memory(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: + """Copy the current top-level memory into this round's own memory dir. + + Called before a round's training pass, so it starts from whatever + memory the previous round left behind (or empty, on round 1). + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number to load memory into. + instance_id : str | None + The eval task's ``task_id``, if running an eval task (see + ``round_dir``). Omit for training-only mode. + + Returns + ------- + Path + This round's own memory dir, ready for the round to use. + """ + src = memory_dir(workdir) + dst = round_memory_dir(workdir, round_num, instance_id=instance_id) + dst.mkdir(parents=True, exist_ok=True) + if src.is_dir(): + shutil.copytree(src, dst, dirs_exist_ok=True) + return dst + + +def save_round_memory(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: + """Copy this round's memory back up to the top-level memory dir. + + Called after a round's training pass, so later rounds (and the + final saved memory) see what this round learned. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number whose memory should be saved. + instance_id : str | None + The eval task's ``task_id``, if running an eval task (see + ``round_dir``). Omit for training-only mode. + + Returns + ------- + Path + The top-level ``memory`` dir, now updated with this round's changes. + """ + src = round_memory_dir(workdir, round_num, instance_id=instance_id) + dst = memory_dir(workdir) + dst.mkdir(parents=True, exist_ok=True) + if src.is_dir(): + shutil.copytree(src, dst, dirs_exist_ok=True) + return dst + + +def round_log_path(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: + """Return the path to a round's training log. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number. + instance_id : str | None + The eval task's ``task_id``, if running an eval task (see + ``round_dir``). Omit for training-only mode. + + Returns + ------- + Path + This round's ``round.log``. + """ + return round_dir(workdir, round_num, instance_id=instance_id) / ROUND_LOG_FILENAME + + +def round_patch_path(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: + """Return the path to a round's captured repo diff. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number. + instance_id : str | None + The eval task's ``task_id``, if running an eval task (see + ``round_dir``). Omit for training-only mode. + + Returns + ------- + Path + This round's ``repo.patch``. + """ + return round_dir(workdir, round_num, instance_id=instance_id) / ROUND_PATCH_FILENAME + + +def eval_dir( + workdir: Path, round_num: int, instance_id: str, *, create: bool = False +) -> Path: + """Return (and optionally create) an eval task instance's eval directory. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number this eval instance belongs to. + instance_id : str + The eval task instance identifier. + create : bool + If True, create the directory (and parents) if missing. + + Returns + ------- + Path + ``workdir/rounds_{instance_id}/round_{round_num}/eval``. + """ + path = round_dir(workdir, round_num, instance_id=instance_id) / EVAL_DIRNAME + if create: + path.mkdir(parents=True, exist_ok=True) + return path + + +def eval_result_path(workdir: Path, round_num: int, instance_id: str) -> Path: + """Return the path to an eval instance's result file. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number this eval instance belongs to. + instance_id : str + The eval task instance identifier. + + Returns + ------- + Path + This eval instance's ``result.json``. + """ + return eval_dir(workdir, round_num, instance_id) / RESULT_FILENAME + + +def eval_log_path(workdir: Path, round_num: int, instance_id: str) -> Path: + """Return the path to an eval instance's log file. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number this eval instance belongs to. + instance_id : str + The eval task instance identifier. + + Returns + ------- + Path + This eval instance's ``eval.log``. + """ + return eval_dir(workdir, round_num, instance_id) / EVAL_LOG_FILENAME + + +def eval_patch_path(workdir: Path, round_num: int, instance_id: str) -> Path: + """Return the path to an eval instance's captured repo diff. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number this eval instance belongs to. + instance_id : str + The eval task instance identifier. + + Returns + ------- + Path + This eval instance's ``repo.patch``. + """ + return eval_dir(workdir, round_num, instance_id) / ROUND_PATCH_FILENAME \ No newline at end of file diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py index 8754211..1835e8d 100644 --- a/test/auto_memory/eval/test_swebenchverified.py +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -13,14 +13,23 @@ from microbots.auto_memory.eval.swebenchverified import ( SweBenchInstance, SweBenchVerifiedTask, + _load_dataset_rows, load_instance_using_id, load_instances_of_repo, ) -from microbots.auto_memory.task import CallbackResult +from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome MODULE = "microbots.auto_memory.eval.swebenchverified" +@pytest.fixture(autouse=True) +def _clear_dataset_cache(): + """Clear ``_load_dataset_rows``'s cache so each test's ``load_dataset`` mock takes effect.""" + _load_dataset_rows.cache_clear() + yield + _load_dataset_rows.cache_clear() + + def _fake_rows(): return [ { @@ -89,6 +98,20 @@ def test_load_instance_using_id_raises_when_not_found(mock_load_dataset): load_instance_using_id("does-not-exist") +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_dataset_rows_are_cached_across_repeated_calls(mock_load_dataset): + """``load_dataset`` should only be called once per ``dataset_name``, even + across multiple ``load_instances_of_repo``/``load_instance_using_id`` calls.""" + mock_load_dataset.return_value = _fake_rows() + + load_instances_of_repo(repo="django/django") + load_instances_of_repo(repo=None) + load_instance_using_id("astropy__astropy-1") + + mock_load_dataset.assert_called_once() + + # --------------------------------------------------------------------------- # SweBenchVerifiedTask.setup / build_prompt / teardown # --------------------------------------------------------------------------- @@ -117,7 +140,32 @@ def test_setup_clones_and_checks_out_base_commit(mock_run): @pytest.mark.unit def test_build_prompt_returns_problem_statement(): task = SweBenchVerifiedTask(_instance()) - assert task.build_prompt("/repo") == "fix the bug" + assert task.build_prompt() == "fix the bug" + + +@pytest.mark.unit +def test_task_id_is_instance_id(): + task = SweBenchVerifiedTask(_instance()) + assert task.task_id == "django__django-1" + + +@pytest.mark.unit +def test_build_result_includes_dataset_fields(): + task = SweBenchVerifiedTask(_instance()) + outcome = EvalOutcome( + passed=True, + output="agent output", + result=CallbackResult(passed=True, reason="resolved"), + log_path="/dev/null", + ) + + assert task.build_result(outcome) == { + "passed": True, + "reason": "resolved", + "instance_id": "django__django-1", + "repo": "django/django", + "base_commit": "abc123", + } @pytest.mark.unit @@ -251,7 +299,7 @@ def test_check_cleans_up_even_when_harness_raises(mock_run, mock_rmtree, tmp_pat @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock_memory_tool): +def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock_memory_tool, tmp_path): from microbots.MicroBot import BotRunResult mock_bot = MagicMock() @@ -261,14 +309,14 @@ def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock task = SweBenchVerifiedTask(_instance()) calls = [] task.setup = lambda repo_path: calls.append(("setup", repo_path)) - task.build_prompt = lambda repo_path: "do the task" + task.build_prompt = lambda: "do the task" task.check = lambda repo_path, agent_output, log_path: ( calls.append(("check", repo_path, agent_output, log_path)) or CallbackResult(passed=True, reason="ok") ) task.teardown = lambda repo_path: calls.append(("teardown", repo_path)) - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert calls[0] == ("setup", "/repo") assert calls[1] == ("check", "/repo", "agent did stuff", outcome.log_path) @@ -280,7 +328,7 @@ def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_tool): +def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_tool, tmp_path): from microbots.MicroBot import BotRunResult mock_bot = MagicMock() @@ -289,7 +337,7 @@ def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_t task = SweBenchVerifiedTask(_instance()) task.setup = lambda repo_path: None - task.build_prompt = lambda repo_path: "do the task" + task.build_prompt = lambda: "do the task" task.teardown = lambda repo_path: None seen_log_exists = {} @@ -299,7 +347,7 @@ def _check(repo_path, agent_output, log_path): task.check = _check - task.run("/repo", "/memory", "azure-openai/gpt-4o") + task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert seen_log_exists["exists"] is True @@ -307,7 +355,7 @@ def _check(repo_path, agent_output, log_path): @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool): +def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool, tmp_path): from microbots.MicroBot import BotRunResult mock_bot = MagicMock() @@ -317,11 +365,11 @@ def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool task = SweBenchVerifiedTask(_instance()) check_calls = [] task.setup = lambda repo_path: None - task.build_prompt = lambda repo_path: "do the task" + task.build_prompt = lambda: "do the task" task.check = lambda *a: check_calls.append(a) task.teardown = lambda repo_path: None - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert check_calls == [] assert outcome.passed is False @@ -331,17 +379,17 @@ def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_converts_build_prompt_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): +def test_run_converts_build_prompt_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool, tmp_path): task = SweBenchVerifiedTask(_instance()) task.setup = lambda repo_path: None task.teardown = lambda repo_path: None - def _build_prompt(repo_path): + def _build_prompt(): raise ValueError("bad prompt") task.build_prompt = _build_prompt - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert outcome.passed is False assert "bad prompt" in outcome.result.reason @@ -352,7 +400,7 @@ def _build_prompt(repo_path): @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): +def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool, tmp_path): from microbots.MicroBot import BotRunResult mock_bot = MagicMock() @@ -361,7 +409,7 @@ def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memor task = SweBenchVerifiedTask(_instance()) task.setup = lambda repo_path: None - task.build_prompt = lambda repo_path: "do the task" + task.build_prompt = lambda: "do the task" task.teardown = lambda repo_path: None def _check(repo_path, agent_output, log_path): @@ -369,7 +417,7 @@ def _check(repo_path, agent_output, log_path): task.check = _check - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert outcome.passed is False assert "check exploded" in outcome.result.reason @@ -378,16 +426,16 @@ def _check(repo_path, agent_output, log_path): @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_still_calls_teardown_when_body_raises(mock_bot_cls, mock_memory_tool): +def test_run_still_calls_teardown_when_body_raises(mock_bot_cls, mock_memory_tool, tmp_path): mock_bot_cls.side_effect = RuntimeError("bot construction failed") task = SweBenchVerifiedTask(_instance()) teardown_calls = [] task.setup = lambda repo_path: None - task.build_prompt = lambda repo_path: "do the task" + task.build_prompt = lambda: "do the task" task.teardown = lambda repo_path: teardown_calls.append(repo_path) - task.run("/repo", "/memory", "azure-openai/gpt-4o") + task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert teardown_calls == ["/repo"] @@ -395,7 +443,7 @@ def test_run_still_calls_teardown_when_body_raises(mock_bot_cls, mock_memory_too @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_teardown_exception_does_not_clobber_returned_outcome(mock_bot_cls, mock_memory_tool): +def test_run_teardown_exception_does_not_clobber_returned_outcome(mock_bot_cls, mock_memory_tool, tmp_path): from microbots.MicroBot import BotRunResult mock_bot = MagicMock() @@ -404,7 +452,7 @@ def test_run_teardown_exception_does_not_clobber_returned_outcome(mock_bot_cls, task = SweBenchVerifiedTask(_instance()) task.setup = lambda repo_path: None - task.build_prompt = lambda repo_path: "do the task" + task.build_prompt = lambda: "do the task" task.check = lambda repo_path, agent_output, log_path: CallbackResult(passed=True, reason="ok") def _teardown(repo_path): @@ -412,7 +460,7 @@ def _teardown(repo_path): task.teardown = _teardown - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) # teardown() raised, but the already-computed EvalOutcome must still be returned assert outcome.passed is True diff --git a/test/auto_memory/test_analyzer.py b/test/auto_memory/test_analyzer.py index 601b044..3566b65 100644 --- a/test/auto_memory/test_analyzer.py +++ b/test/auto_memory/test_analyzer.py @@ -9,7 +9,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) from microbots.auto_memory.analyzer import build_feedback -from microbots.auto_memory.task import CallbackResult, EvalOutcome +from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome from microbots.MicroBot import BotRunResult diff --git a/test/auto_memory/test_cli.py b/test/auto_memory/test_cli.py index da3ca8b..6fcf716 100644 --- a/test/auto_memory/test_cli.py +++ b/test/auto_memory/test_cli.py @@ -2,6 +2,7 @@ import os import sys +from pathlib import Path from unittest.mock import MagicMock, patch import pytest @@ -12,27 +13,25 @@ MODULE = "microbots.auto_memory.cli" -BASE_ARGS = ["--repo", "/repo", "--memory-dir", "/memory", "--model", "azure-openai/gpt-4o"] +BASE_ARGS = ["--model", "azure-openai/gpt-4o"] +FAKE_WORKDIR = Path("/workdir") @pytest.mark.unit def test_parse_args_defaults(): args = parse_args(BASE_ARGS) - assert args.repo == "/repo" - assert args.memory_dir == "/memory" assert args.model == "azure-openai/gpt-4o" assert args.task is None assert args.max_rounds == 5 - assert args.training_iterations == 1 + assert args.training_iterations == 10 @pytest.mark.unit -def test_parse_args_with_known_task_adds_its_flags(): - args = parse_args(BASE_ARGS + ["--task", "swebenchverified", "--instance-id", "django__django-1"]) +def test_parse_args_accepts_known_task(): + args = parse_args(BASE_ARGS + ["--task", "swebenchverified"]) assert args.task == "swebenchverified" - assert args.instance_id == "django__django-1" @pytest.mark.unit @@ -42,67 +41,107 @@ def test_parse_args_rejects_unknown_task(): @pytest.mark.unit -@patch(f"{MODULE}.run_training_loop") -def test_main_runs_training_only_when_task_omitted(mock_run_training_loop): +def test_parse_args_workdir_defaults_to_none(): + args = parse_args(BASE_ARGS) + + assert args.workdir is None + + +@pytest.mark.unit +def test_parse_args_picks_up_explicit_workdir(): + args = parse_args(BASE_ARGS + ["--workdir", "/custom/workdir"]) + + assert args.workdir == "/custom/workdir" + + +@pytest.mark.unit +@patch(f"{MODULE}.require_workdir") +@patch(f"{MODULE}.resolve_workdir") +@patch(f"{MODULE}.load_config", return_value={}) +@patch(f"{MODULE}.run") +def test_main_uses_explicit_workdir_over_resolve_workdir( + mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir +): + main(BASE_ARGS + ["--workdir", "/custom/workdir"]) + + mock_resolve_workdir.assert_not_called() + mock_require_workdir.assert_called_once_with(Path("/custom/workdir")) + mock_run.assert_called_once_with( + workdir=Path("/custom/workdir"), + model="azure-openai/gpt-4o", + task=None, + max_rounds=5, + training_iterations=10, + config={}, + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.require_workdir") +@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR) +@patch(f"{MODULE}.load_config", return_value={}) +@patch(f"{MODULE}.run") +def test_main_falls_back_to_resolve_workdir_when_not_given( + mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir +): main(BASE_ARGS) - mock_run_training_loop.assert_called_once_with( - repo_path="/repo", - feedback="", - memory_dir="/memory", + mock_resolve_workdir.assert_called_once_with() + mock_require_workdir.assert_called_once_with(FAKE_WORKDIR) + mock_run.assert_called_once_with( + workdir=FAKE_WORKDIR, model="azure-openai/gpt-4o", - iterations=1, + task=None, + max_rounds=5, + training_iterations=10, + config={}, ) @pytest.mark.unit -@patch(f"{MODULE}.run_train_eval_loop") -@patch(f"{MODULE}.run_training_loop") -def test_main_does_not_run_eval_loop_when_task_omitted(mock_run_training_loop, mock_run_train_eval_loop): +@patch(f"{MODULE}.require_workdir") +@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR) +@patch(f"{MODULE}.run") +def test_main_calls_run_with_task_none_when_task_omitted(mock_run, mock_resolve_workdir, mock_require_workdir): main(BASE_ARGS) - mock_run_train_eval_loop.assert_not_called() + assert mock_run.call_args.kwargs["task"] is None @pytest.mark.unit -@patch(f"{MODULE}.run_train_eval_loop") -def test_main_runs_eval_loop_for_each_task_when_task_given(mock_run_train_eval_loop): +@patch(f"{MODULE}.require_workdir") +@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR) +@patch(f"{MODULE}.load_config", return_value={}) +@patch(f"{MODULE}.run") +def test_main_calls_run_for_each_task_when_task_given( + mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir +): fake_task = MagicMock() - mock_run_train_eval_loop.return_value = MagicMock(passed=True, rounds_run=1) + mock_run.return_value = MagicMock(passed=True, rounds_run=1) - with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_cli_args=lambda args: [fake_task])}): + with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_config=lambda task_args: [fake_task])}): main(BASE_ARGS + ["--task", "swebenchverified"]) - mock_run_train_eval_loop.assert_called_once_with( - repo_path="/repo", - memory_dir="/memory", + mock_run.assert_called_once_with( + workdir=FAKE_WORKDIR, model="azure-openai/gpt-4o", task=fake_task, max_rounds=5, - training_iterations=1, + training_iterations=10, + config={}, ) @pytest.mark.unit -@patch(f"{MODULE}.run_training_loop") -def test_main_does_not_run_training_only_path_when_task_given(mock_run_training_loop): - fake_task = MagicMock() - - with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_cli_args=lambda args: [fake_task])}): - with patch(f"{MODULE}.run_train_eval_loop") as mock_run_train_eval_loop: - mock_run_train_eval_loop.return_value = MagicMock(passed=True, rounds_run=1) - main(BASE_ARGS + ["--task", "swebenchverified"]) - - mock_run_training_loop.assert_not_called() - - -@pytest.mark.unit -@patch(f"{MODULE}.run_train_eval_loop") -def test_main_runs_eval_loop_once_per_returned_task(mock_run_train_eval_loop): +@patch(f"{MODULE}.require_workdir") +@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR) +@patch(f"{MODULE}.load_config", return_value={}) +@patch(f"{MODULE}.run") +def test_main_runs_once_per_returned_task(mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir): fake_tasks = [MagicMock(), MagicMock()] - mock_run_train_eval_loop.return_value = MagicMock(passed=False, rounds_run=5) + mock_run.return_value = MagicMock(passed=False, rounds_run=5) - with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_cli_args=lambda args: fake_tasks)}): + with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_config=lambda task_args: fake_tasks)}): main(BASE_ARGS + ["--task", "swebenchverified"]) - assert mock_run_train_eval_loop.call_count == 2 + assert mock_run.call_count == 2 diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py index e869c55..10867f2 100644 --- a/test/auto_memory/test_orchestrator.py +++ b/test/auto_memory/test_orchestrator.py @@ -1,16 +1,28 @@ """Unit tests for microbots.auto_memory.orchestrator.""" +import json import os import sys from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch import pytest sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) -from microbots.auto_memory.orchestrator import LoopResult, run_train_eval_loop, run_training_loop -from microbots.auto_memory.task import CallbackResult, EvalOutcome +from microbots.auto_memory.orchestrator import ( + LoopResult, + clone_repo, + reset_repo, + run, + run_train_eval_loop, + run_training_loop, + write_eval_result, +) +from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome +from microbots.auto_memory.workdir import eval_result_path, memory_dir, round_memory_dir + +MODULE = "microbots.auto_memory.orchestrator" def _make_outcome(passed: bool, log_path: str, reason: str = "reason") -> EvalOutcome: @@ -27,15 +39,26 @@ def _touch(path: str) -> str: return path +def _make_task() -> MagicMock: + """A MagicMock task with a real-ish task_id/build_result, for round tests.""" + task = MagicMock() + task.task_id = "task-1" + task.build_result.side_effect = lambda outcome: { + "passed": outcome.result.passed, + "reason": outcome.result.reason, + } + return task + + @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, mock_run_training_loop, tmp_path): log_path = _touch(str(tmp_path / "round1.log")) - task = MagicMock() + task = _make_task() task.run.return_value = _make_outcome(passed=True, log_path=log_path) - result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert isinstance(result, LoopResult) assert result.passed is True @@ -51,20 +74,24 @@ def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, m def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) - task = MagicMock() + task = _make_task() task.run.side_effect = [ _make_outcome(passed=False, log_path=log1), _make_outcome(passed=True, log_path=log2), ] mock_build_feedback.return_value = "feedback text" - result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert result.passed is True assert result.rounds_run == 2 mock_build_feedback.assert_called_once() mock_run_training_loop.assert_called_once_with( - repo_path="/repo", feedback="feedback text", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=1 + repo_path="/repo", + feedback="feedback text", + memory_dir=str(round_memory_dir(tmp_path, 1, instance_id="task-1")), + model="azure-openai/gpt-4o", + iterations=10, ) @@ -72,14 +99,14 @@ def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, @patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_training_loop, tmp_path): - task = MagicMock() + task = _make_task() task.run.side_effect = [ _make_outcome(passed=False, log_path=_touch(str(tmp_path / f"round{i}.log"))) for i in range(3) ] mock_build_feedback.return_value = "feedback text" - result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=3) + result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=3) assert result.passed is False assert result.rounds_run == 3 @@ -92,33 +119,33 @@ def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_ @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_log_path_deleted_after_passing_round(mock_build_feedback, mock_run_training_loop, tmp_path): +def test_log_path_persists_after_passing_round(mock_build_feedback, mock_run_training_loop, tmp_path): log_path = _touch(str(tmp_path / "round1.log")) - task = MagicMock() + task = _make_task() task.run.return_value = _make_outcome(passed=True, log_path=log_path) - run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) - assert not Path(log_path).exists() + assert Path(log_path).exists() @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_log_path_deleted_after_failing_round(mock_build_feedback, mock_run_training_loop, tmp_path): +def test_log_path_persists_after_failing_round(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) - task = MagicMock() + task = _make_task() task.run.side_effect = [ _make_outcome(passed=False, log_path=log1), _make_outcome(passed=True, log_path=log2), ] mock_build_feedback.return_value = "feedback text" - run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) - assert not Path(log1).exists() - assert not Path(log2).exists() + assert Path(log1).exists() + assert Path(log2).exists() @pytest.mark.unit @@ -127,19 +154,19 @@ def test_log_path_deleted_after_failing_round(mock_build_feedback, mock_run_trai def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) - task = MagicMock() + task = _make_task() task.run.side_effect = [ _make_outcome(passed=False, log_path=log1), _make_outcome(passed=True, log_path=log2), ] mock_build_feedback.side_effect = RuntimeError("analysis bot crashed") - result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert result.passed is True assert result.rounds_run == 2 mock_run_training_loop.assert_not_called() - assert not Path(log1).exists() + assert Path(log1).exists() @pytest.mark.unit @@ -148,7 +175,7 @@ def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_ def test_loop_forwards_training_iterations_to_run_training_loop(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) - task = MagicMock() + task = _make_task() task.run.side_effect = [ _make_outcome(passed=False, log_path=log1), _make_outcome(passed=True, log_path=log2), @@ -156,11 +183,15 @@ def test_loop_forwards_training_iterations_to_run_training_loop(mock_build_feedb mock_build_feedback.return_value = "feedback text" run_train_eval_loop( - "/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5, training_iterations=4 + "/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5, training_iterations=4 ) mock_run_training_loop.assert_called_once_with( - repo_path="/repo", feedback="feedback text", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=4 + repo_path="/repo", + feedback="feedback text", + memory_dir=str(round_memory_dir(tmp_path, 1, instance_id="task-1")), + model="azure-openai/gpt-4o", + iterations=4, ) @@ -170,7 +201,7 @@ def test_loop_forwards_training_iterations_to_run_training_loop(mock_build_feedb def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) - task = MagicMock() + task = _make_task() task.run.side_effect = [ _make_outcome(passed=False, log_path=log1), _make_outcome(passed=True, log_path=log2), @@ -178,19 +209,20 @@ def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_ru mock_build_feedback.return_value = "feedback text" mock_run_training_loop.side_effect = RuntimeError("training crashed") - result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert result.passed is True assert result.rounds_run == 2 - assert not Path(log1).exists() + assert Path(log1).exists() @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training") -def test_run_training_loop_calls_run_training_once_by_default(mock_run_training): +def test_run_training_loop_calls_run_training_ten_times_by_default(mock_run_training): run_training_loop(repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o") - mock_run_training.assert_called_once_with( + assert mock_run_training.call_count == 10 + mock_run_training.assert_called_with( repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o" ) @@ -217,3 +249,206 @@ def test_run_training_loop_reuses_same_memory_dir_each_pass(mock_run_training): memory_dirs = {call.kwargs["memory_dir"] for call in mock_run_training.call_args_list} assert memory_dirs == {"/memory"} + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_clone_repo_clones_when_missing(mock_run, tmp_path): + repo_path = tmp_path / "repo" + + clone_repo("https://example.com/repo.git", repo_path) + + mock_run.assert_called_once_with( + ["git", "clone", "https://example.com/repo.git", str(repo_path)], check=True + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_clone_repo_is_noop_when_already_present(mock_run, tmp_path): + repo_path = tmp_path / "repo" + repo_path.mkdir() + + clone_repo("https://example.com/repo.git", repo_path) + + mock_run.assert_not_called() + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_reset_repo_runs_hard_reset_then_clean(mock_run, tmp_path): + repo_path = tmp_path / "repo" + + reset_repo(repo_path, "abc123") + + assert mock_run.call_args_list == [ + call(["git", "reset", "--hard", "abc123"], cwd=repo_path, check=True), + call(["git", "clean", "-fd"], cwd=repo_path, check=True), + ] + + +@pytest.mark.unit +def test_write_eval_result_writes_task_build_result_as_json(tmp_path): + task = MagicMock() + task.task_id = "django__django-1" + task.build_result.return_value = {"passed": True, "reason": "resolved"} + outcome = _make_outcome(passed=True, log_path="/dev/null") + + write_eval_result(tmp_path, 2, task, outcome) + + result_path = eval_result_path(tmp_path, 2, "django__django-1") + assert json.loads(result_path.read_text()) == {"passed": True, "reason": "resolved"} + task.build_result.assert_called_once_with(outcome) + + +@pytest.mark.unit +def test_write_eval_result_creates_missing_parent_dirs(tmp_path): + task = MagicMock() + task.task_id = "some-task" + task.build_result.return_value = {"passed": False, "reason": "nope"} + outcome = _make_outcome(passed=False, log_path="/dev/null") + + write_eval_result(tmp_path, 1, task, outcome) + + assert eval_result_path(tmp_path, 1, "some-task").exists() + + +@pytest.mark.unit +@patch(f"{MODULE}.build_feedback") +def test_loop_writes_eval_result_for_every_round(mock_build_feedback, tmp_path): + log1 = _touch(str(tmp_path / "round1.log")) + log2 = _touch(str(tmp_path / "round2.log")) + task = _make_task() + task.run.side_effect = [ + _make_outcome(passed=False, log_path=log1), + _make_outcome(passed=True, log_path=log2), + ] + mock_build_feedback.return_value = "feedback text" + + with patch(f"{MODULE}.run_training_loop"): + run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + + assert eval_result_path(tmp_path, 1, "task-1").exists() + assert eval_result_path(tmp_path, 2, "task-1").exists() + assert json.loads(eval_result_path(tmp_path, 2, "task-1").read_text()) == { + "passed": True, + "reason": "reason", + } + + +@pytest.mark.unit +@patch(f"{MODULE}.run_training_loop") +def test_run_calls_run_training_loop_when_task_is_none(mock_run_training_loop, tmp_path): + result = run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None, training_iterations=2) + + mock_run_training_loop.assert_called_once_with( + repo_path=str(tmp_path / "repo"), + feedback="", + memory_dir=str(round_memory_dir(tmp_path, 1)), + model="azure-openai/gpt-4o", + iterations=2, + ) + assert result is None + + +@pytest.mark.unit +@patch(f"{MODULE}.run_train_eval_loop") +def test_run_calls_run_train_eval_loop_when_task_given(mock_run_train_eval_loop, tmp_path): + fake_task = MagicMock() + mock_run_train_eval_loop.return_value = "loop-result" + + result = run( + workdir=tmp_path, + model="azure-openai/gpt-4o", + task=fake_task, + max_rounds=3, + training_iterations=2, + ) + + mock_run_train_eval_loop.assert_called_once_with( + repo_path=str(tmp_path / "repo"), + workdir=tmp_path, + model="azure-openai/gpt-4o", + task=fake_task, + max_rounds=3, + training_iterations=2, + ) + assert result == "loop-result" + + +@pytest.mark.unit +@patch(f"{MODULE}.run_train_eval_loop") +@patch(f"{MODULE}.run_training_loop") +def test_run_does_not_call_eval_loop_when_task_is_none(mock_run_training_loop, mock_run_train_eval_loop, tmp_path): + run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) + + mock_run_train_eval_loop.assert_not_called() + + +@pytest.mark.unit +@patch(f"{MODULE}.run_train_eval_loop") +@patch(f"{MODULE}.run_training_loop") +def test_run_does_not_call_training_loop_when_task_given(mock_run_training_loop, mock_run_train_eval_loop, tmp_path): + run(workdir=tmp_path, model="azure-openai/gpt-4o", task=MagicMock()) + + mock_run_training_loop.assert_not_called() + + +@pytest.mark.unit +@patch(f"{MODULE}.clone_repo") +@patch(f"{MODULE}.run_training_loop") +def test_run_clones_repo_from_config_when_repo_url_given(mock_run_training_loop, mock_clone_repo, tmp_path): + (tmp_path / "config.yaml").write_text("repo: https://example.com/repo.git\n") + + run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) + + mock_clone_repo.assert_called_once_with("https://example.com/repo.git", tmp_path / "repo") + + +@pytest.mark.unit +@patch(f"{MODULE}.clone_repo") +@patch(f"{MODULE}.run_training_loop") +def test_run_does_not_clone_when_config_has_no_repo(mock_run_training_loop, mock_clone_repo, tmp_path): + run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) + + mock_clone_repo.assert_not_called() + + +@pytest.mark.unit +@patch(f"{MODULE}.run_training_loop") +def test_run_promotes_round1_memory_to_top_level_for_train_only_mode(mock_run_training_loop, tmp_path): + def fake_train(repo_path, feedback, memory_dir, model, iterations=1): + Path(memory_dir, "notes.md").write_text("learned something") + + mock_run_training_loop.side_effect = fake_train + + run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) + + assert (memory_dir(tmp_path) / "notes.md").read_text() == "learned something" + + +@pytest.mark.unit +@patch(f"{MODULE}.build_feedback") +@patch(f"{MODULE}.run_training_loop") +def test_loop_carries_memory_forward_between_rounds(mock_run_training_loop, mock_build_feedback, tmp_path): + log1 = _touch(str(tmp_path / "round1.log")) + log2 = _touch(str(tmp_path / "round2.log")) + mock_build_feedback.return_value = "feedback text" + seen_memory_dirs = [] + + def fake_run(repo_path, memory_dir, model, log_path): + round_num = len(seen_memory_dirs) + 1 + if round_num == 2: + # Round 2 should start with whatever round 1 saved. + assert (Path(memory_dir) / "notes.md").read_text() == "round 1 progress" + seen_memory_dirs.append(memory_dir) + Path(memory_dir, "notes.md").write_text(f"round {round_num} progress") + log_path = log1 if round_num == 1 else log2 + return _make_outcome(passed=round_num == 2, log_path=log_path) + + task = _make_task() + task.run.side_effect = fake_run + + run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + + assert (memory_dir(tmp_path) / "notes.md").read_text() == "round 2 progress" diff --git a/test/auto_memory/test_task.py b/test/auto_memory/test_task.py index aee0f9d..ed11bb2 100644 --- a/test/auto_memory/test_task.py +++ b/test/auto_memory/test_task.py @@ -1,4 +1,4 @@ -"""Unit tests for microbots.auto_memory.task.""" +"""Unit tests for microbots.auto_memory.evalTask.""" import os import sys @@ -7,13 +7,13 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) -from microbots.auto_memory.task import CallbackResult, EvalOutcome, EvalTask +from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome, EvalTask class _RunOnlyTask(EvalTask): """A task that overrides only run(), never touching the optional hooks.""" - def run(self, repo_path, memory_dir, model): + def run(self, repo_path, memory_dir, model, log_path): return EvalOutcome( passed=True, output="custom output", @@ -31,7 +31,7 @@ def test_run_is_abstract(): @pytest.mark.unit def test_subclass_overriding_only_run_is_instantiable(): task = _RunOnlyTask() - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", "/log") assert outcome.passed is True assert outcome.output == "custom output" @@ -51,7 +51,7 @@ def test_default_teardown_is_a_noop(): @pytest.mark.unit def test_default_build_prompt_returns_empty_string(): - assert _RunOnlyTask().build_prompt("/repo") == "" + assert _RunOnlyTask().build_prompt() == "" @pytest.mark.unit @@ -60,3 +60,20 @@ def test_default_check_passes_by_default(): assert isinstance(result, CallbackResult) assert result.passed is True + + +@pytest.mark.unit +def test_default_task_id_is_class_name(): + assert _RunOnlyTask().task_id == "_RunOnlyTask" + + +@pytest.mark.unit +def test_default_build_result_returns_passed_and_reason(): + outcome = EvalOutcome( + passed=False, + output="agent output", + result=CallbackResult(passed=False, reason="check failed"), + log_path="/dev/null", + ) + + assert _RunOnlyTask().build_result(outcome) == {"passed": False, "reason": "check failed"} diff --git a/test/auto_memory/test_task_registry.py b/test/auto_memory/test_task_registry.py index d06247f..4caa88f 100644 --- a/test/auto_memory/test_task_registry.py +++ b/test/auto_memory/test_task_registry.py @@ -9,7 +9,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) -from microbots.auto_memory.task import EvalTask +from microbots.auto_memory.evalTask import EvalTask from microbots.auto_memory.task_registry import TASK_REGISTRY, create_task, discover_tasks, register_task MODULE_PATH = "microbots.auto_memory.task_registry" diff --git a/test/auto_memory/test_workdir.py b/test/auto_memory/test_workdir.py new file mode 100644 index 0000000..f899588 --- /dev/null +++ b/test/auto_memory/test_workdir.py @@ -0,0 +1,94 @@ +"""Unit tests for microbots.auto_memory.workdir.""" + +import os +import sys + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) + +from microbots.auto_memory.workdir import ( + CONFIG_FILENAME, + load_config, + load_round_memory, + memory_dir, + round_memory_dir, + save_round_memory, +) + + +@pytest.mark.unit +def test_load_config_returns_empty_dict_when_file_missing(tmp_path): + assert load_config(tmp_path) == {} + + +@pytest.mark.unit +def test_load_config_returns_empty_dict_when_file_empty(tmp_path): + (tmp_path / CONFIG_FILENAME).write_text("") + + assert load_config(tmp_path) == {} + + +@pytest.mark.unit +def test_load_config_parses_yaml_contents(tmp_path): + (tmp_path / CONFIG_FILENAME).write_text("repo: https://example.com/repo.git\ntask: swebenchverified\n") + + assert load_config(tmp_path) == { + "repo": "https://example.com/repo.git", + "task": "swebenchverified", + } + + +@pytest.mark.unit +def test_load_round_memory_creates_empty_dir_when_no_top_level_memory(tmp_path): + result = load_round_memory(tmp_path, 1) + + assert result == round_memory_dir(tmp_path, 1) + assert result.is_dir() + assert list(result.iterdir()) == [] + + +@pytest.mark.unit +def test_load_round_memory_copies_top_level_memory_into_round(tmp_path): + top_memory = memory_dir(tmp_path) + top_memory.mkdir(parents=True) + (top_memory / "notes.md").write_text("prior findings") + + result = load_round_memory(tmp_path, 2) + + assert (result / "notes.md").read_text() == "prior findings" + + +@pytest.mark.unit +def test_save_round_memory_creates_empty_dir_when_no_round_memory(tmp_path): + result = save_round_memory(tmp_path, 1) + + assert result == memory_dir(tmp_path) + assert result.is_dir() + assert list(result.iterdir()) == [] + + +@pytest.mark.unit +def test_save_round_memory_copies_round_memory_to_top_level(tmp_path): + round_memory = round_memory_dir(tmp_path, 3) + round_memory.mkdir(parents=True) + (round_memory / "learned.md").write_text("new insight") + + result = save_round_memory(tmp_path, 3) + + assert (result / "learned.md").read_text() == "new insight" + + +@pytest.mark.unit +def test_save_round_memory_overwrites_stale_top_level_files(tmp_path): + top_memory = memory_dir(tmp_path) + top_memory.mkdir(parents=True) + (top_memory / "notes.md").write_text("old") + + round_memory = round_memory_dir(tmp_path, 1) + round_memory.mkdir(parents=True) + (round_memory / "notes.md").write_text("new") + + save_round_memory(tmp_path, 1) + + assert (top_memory / "notes.md").read_text() == "new" From 085f462492e15397f14ea1d41c2138e029599b74 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 2 Sep 2026 07:27:22 +0000 Subject: [PATCH 08/21] Update development mode package installation to include training dependencies --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e898772..a87c7a7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -165,7 +165,7 @@ jobs: - name: Install package in development mode run: | - pip install -e . + pip install -e ".[training]" - name: Install GitHub Copilot SDK dependencies for GHCP tests if: matrix.test-type == 'ghcp' From 25e0b9186d1cf870b65af2fce58607a134f27f67 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 2 Sep 2026 07:43:17 +0000 Subject: [PATCH 09/21] Add unit tests --- .../auto_memory/eval/test_swebenchverified.py | 42 ++++++ test/auto_memory/test_workdir.py | 124 ++++++++++++++++++ 2 files changed, 166 insertions(+) diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py index 1835e8d..98a89cc 100644 --- a/test/auto_memory/eval/test_swebenchverified.py +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -523,3 +523,45 @@ class _EmptyArgs: mock_load_instances_of_repo.assert_called_once_with(repo=None) assert len(tasks) == 1 + + +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.from_config +# --------------------------------------------------------------------------- + +@pytest.mark.unit +@patch(f"{MODULE}.load_instance_using_id") +def test_from_config_uses_instance_id_when_given(mock_load_instance_using_id): + mock_load_instance_using_id.return_value = _instance() + + tasks = SweBenchVerifiedTask.from_config( + {"instance_id": "django__django-1", "swebench_repo": None} + ) + + mock_load_instance_using_id.assert_called_once_with("django__django-1") + assert len(tasks) == 1 + assert isinstance(tasks[0], SweBenchVerifiedTask) + assert tasks[0].instance == _instance() + + +@pytest.mark.unit +@patch(f"{MODULE}.load_instances_of_repo") +def test_from_config_falls_back_to_repo_filter_when_no_instance_id(mock_load_instances_of_repo): + mock_load_instances_of_repo.return_value = [_instance(), _instance()] + + tasks = SweBenchVerifiedTask.from_config({"swebench_repo": "django/django"}) + + mock_load_instances_of_repo.assert_called_once_with(repo="django/django") + assert len(tasks) == 2 + assert all(isinstance(t, SweBenchVerifiedTask) for t in tasks) + + +@pytest.mark.unit +@patch(f"{MODULE}.load_instances_of_repo") +def test_from_config_handles_empty_dict_gracefully(mock_load_instances_of_repo): + mock_load_instances_of_repo.return_value = [_instance()] + + tasks = SweBenchVerifiedTask.from_config({}) + + mock_load_instances_of_repo.assert_called_once_with(repo=None) + assert len(tasks) == 1 diff --git a/test/auto_memory/test_workdir.py b/test/auto_memory/test_workdir.py index f899588..149b223 100644 --- a/test/auto_memory/test_workdir.py +++ b/test/auto_memory/test_workdir.py @@ -9,10 +9,21 @@ from microbots.auto_memory.workdir import ( CONFIG_FILENAME, + eval_dir, + eval_log_path, + eval_patch_path, + eval_result_path, load_config, load_round_memory, memory_dir, + repo_dir, + require_workdir, + resolve_workdir, + round_dir, + round_log_path, round_memory_dir, + round_patch_path, + run_log_path, save_round_memory, ) @@ -92,3 +103,116 @@ def test_save_round_memory_overwrites_stale_top_level_files(tmp_path): save_round_memory(tmp_path, 1) assert (top_memory / "notes.md").read_text() == "new" + + +@pytest.mark.unit +def test_resolve_workdir_defaults_to_cwd(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + + assert resolve_workdir() == tmp_path / "workdir" + + +@pytest.mark.unit +def test_resolve_workdir_uses_given_base(tmp_path): + assert resolve_workdir(tmp_path) == tmp_path / "workdir" + + +@pytest.mark.unit +def test_require_workdir_raises_when_missing(tmp_path): + missing = tmp_path / "nope" + + with pytest.raises(FileNotFoundError): + require_workdir(missing) + + +@pytest.mark.unit +def test_require_workdir_passes_when_present(tmp_path): + require_workdir(tmp_path) + + +@pytest.mark.unit +def test_repo_dir_returns_workdir_repo(tmp_path): + assert repo_dir(tmp_path) == tmp_path / "repo" + + +@pytest.mark.unit +def test_run_log_path_returns_workdir_run_log(tmp_path): + assert run_log_path(tmp_path) == tmp_path / "run.log" + + +@pytest.mark.unit +def test_round_dir_creates_directory_when_requested(tmp_path): + path = round_dir(tmp_path, 1, create=True) + + assert path == tmp_path / "rounds" / "round_1" + assert path.is_dir() + + +@pytest.mark.unit +def test_round_dir_does_not_create_directory_by_default(tmp_path): + path = round_dir(tmp_path, 1) + + assert path == tmp_path / "rounds" / "round_1" + assert not path.exists() + + +@pytest.mark.unit +def test_round_dir_uses_per_instance_dir_when_instance_id_given(tmp_path): + path = round_dir(tmp_path, 1, instance_id="task-1") + + assert path == tmp_path / "rounds_task-1" / "round_1" + + +@pytest.mark.unit +def test_round_log_path_returns_round_log(tmp_path): + assert round_log_path(tmp_path, 2) == round_dir(tmp_path, 2) / "round.log" + + +@pytest.mark.unit +def test_round_log_path_with_instance_id(tmp_path): + assert round_log_path(tmp_path, 2, instance_id="task-1") == round_dir( + tmp_path, 2, instance_id="task-1" + ) / "round.log" + + +@pytest.mark.unit +def test_round_patch_path_returns_repo_patch(tmp_path): + assert round_patch_path(tmp_path, 2) == round_dir(tmp_path, 2) / "repo.patch" + + +@pytest.mark.unit +def test_round_patch_path_with_instance_id(tmp_path): + assert round_patch_path(tmp_path, 2, instance_id="task-1") == round_dir( + tmp_path, 2, instance_id="task-1" + ) / "repo.patch" + + +@pytest.mark.unit +def test_eval_dir_creates_directory_when_requested(tmp_path): + path = eval_dir(tmp_path, 1, "task-1", create=True) + + assert path == tmp_path / "rounds_task-1" / "round_1" / "eval" + assert path.is_dir() + + +@pytest.mark.unit +def test_eval_dir_does_not_create_directory_by_default(tmp_path): + path = eval_dir(tmp_path, 1, "task-1") + + assert path == tmp_path / "rounds_task-1" / "round_1" / "eval" + assert not path.exists() + + +@pytest.mark.unit +def test_eval_result_path_returns_result_json(tmp_path): + assert eval_result_path(tmp_path, 1, "task-1") == eval_dir(tmp_path, 1, "task-1") / "result.json" + + +@pytest.mark.unit +def test_eval_log_path_returns_eval_log(tmp_path): + assert eval_log_path(tmp_path, 1, "task-1") == eval_dir(tmp_path, 1, "task-1") / "eval.log" + + +@pytest.mark.unit +def test_eval_patch_path_returns_repo_patch(tmp_path): + assert eval_patch_path(tmp_path, 1, "task-1") == eval_dir(tmp_path, 1, "task-1") / "repo.patch" From 00d095bef39a81b00e6fe148aec0015ed80c9646 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 2 Sep 2026 12:39:04 +0000 Subject: [PATCH 10/21] Refactor evalTask and related modules: implement build_feedback method, remove analyzer, and update tests --- src/microbots/auto_memory/analyzer.py | 65 -------- .../auto_memory/eval/swebenchverified.py | 96 ++++++------ src/microbots/auto_memory/evalTask.py | 35 ++++- src/microbots/auto_memory/orchestrator.py | 8 +- src/microbots/auto_memory/workdir.py | 21 --- .../auto_memory/eval/test_swebenchverified.py | 142 ++++++++++-------- test/auto_memory/test_analyzer.py | 81 ---------- test/auto_memory/test_orchestrator.py | 103 +++++-------- test/auto_memory/test_task.py | 17 ++- test/auto_memory/test_task_registry.py | 3 + test/auto_memory/test_workdir.py | 13 -- 11 files changed, 221 insertions(+), 363 deletions(-) delete mode 100644 src/microbots/auto_memory/analyzer.py delete mode 100644 test/auto_memory/test_analyzer.py diff --git a/src/microbots/auto_memory/analyzer.py b/src/microbots/auto_memory/analyzer.py deleted file mode 100644 index ca35c16..0000000 --- a/src/microbots/auto_memory/analyzer.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Build feedback text for a failed evaluation round. - -Uses ``LogAnalysisBot`` to analyze the eval callback's raw log and produce -concrete feedback describing what went wrong, to be passed into the -training agent as ``feedback`` for the next round. -""" - -from logging import getLogger - -from microbots.auto_memory.evalTask import EvalOutcome, EvalTask -from microbots.bot.LogAnalysisBot import LogAnalysisBot -from microbots.MicroBot import BotRunResult - -logger = getLogger(__name__) -#make this abstract -def build_feedback( - task: EvalTask, - outcome: EvalOutcome, - repo_path: str, - model: str, -) -> str: - """Analyze a failed eval outcome's log and produce training feedback. - - Parameters - ---------- - task : EvalTask - The eval task that produced ``outcome``. - outcome : EvalOutcome - The failed outcome to analyze, including its ``log_path``. - repo_path : str - Absolute path to the repo the task was evaluated against. - model : str - The model to use, in the format ``/``. - - Returns - ------- - str - Feedback text describing the root cause of the failure and what - the agent's memory notes should cover next time, suitable for - passing as ``feedback`` to ``run_training``. - """ - bot = LogAnalysisBot(model=model, folder_to_mount=repo_path) - result: BotRunResult = bot.run( - file_name=outcome.log_path, - user_prompt=( - "This log was produced while verifying whether an " - "agent completed its task correctly. Identify " - "the root cause of the failure and describe concretely " - "what the agent's memory notes should cover next time to " - "avoid this failure." - ), - ) - - if result.status and result.result: - return result.result - - logger.warning( - "LogAnalysisBot failed to analyze failure (%s); falling back to plain feedback", - result.error, - ) - return ( - f"Evaluation failed. Agent output: {outcome.output}\n" - f"Callback reason: {outcome.result.reason}" - ) - \ No newline at end of file diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index 96017b8..2fd64b1 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -5,7 +5,6 @@ verifies the result via ``swebench.harness.run_evaluation``. """ -import argparse import json import shutil import subprocess @@ -20,7 +19,9 @@ from datasets import load_dataset from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome, EvalTask from microbots.auto_memory.task_registry import register_task +from microbots.bot.LogAnalysisBot import LogAnalysisBot from microbots.bot.WritingBot import WritingBot +from microbots.MicroBot import BotRunResult from microbots.tools.tool_definitions.memory_tool import MemoryTool logger = getLogger(__name__) @@ -168,47 +169,6 @@ def __init__(self, instance: SweBenchInstance | None = None): """ self.instance = instance - @staticmethod - def add_cli_args(parser: argparse.ArgumentParser) -> None: - """Register this task's CLI flags on ``parser``. - - Parameters - ---------- - parser : argparse.ArgumentParser - The CLI's argument parser to add task-specific flags to. - """ - parser.add_argument( - "--instance-id", - help='SWE-bench-verified instance ID, e.g. "django__django-11099".', - ) - parser.add_argument( - "--swebench-repo", - help='Restrict to instances for this repo, e.g. "django/django". ' - "Ignored if --instance-id is given.", - ) - - @classmethod - def from_cli_args(cls, args: argparse.Namespace) -> list["SweBenchVerifiedTask"]: - """Build task(s) from parsed CLI args. - - Parameters - ---------- - args : argparse.Namespace - Parsed CLI args, expected to include ``instance_id`` and/or - ``swebench_repo`` (see ``add_cli_args``). - - Returns - ------- - list[SweBenchVerifiedTask] - One task per matching dataset instance. A single-element - list when ``--instance-id`` is given. - """ - if getattr(args, "instance_id", None): - instances = [load_instance_using_id(args.instance_id)] - else: - instances = load_instances_of_repo(repo=getattr(args, "swebench_repo", None)) - return [cls(instance) for instance in instances] - @classmethod def from_config(cls, task_args: dict) -> list["SweBenchVerifiedTask"]: """Build task(s) from a config's ``task_args`` dict. @@ -217,8 +177,7 @@ def from_config(cls, task_args: dict) -> list["SweBenchVerifiedTask"]: ---------- task_args : dict Task-specific config values, expected to include - ``instance_id`` and/or ``swebench_repo`` (mirrors - ``add_cli_args``'s flags). + ``instance_id`` and/or ``swebench_repo``. Returns ------- @@ -365,6 +324,51 @@ def teardown(self, repo_path: str) -> None: """ subprocess.run(["rm", "-rf", repo_path], check=False) + def build_feedback(self, outcome: EvalOutcome, repo_path: str, model: str, log_path: str) -> str: + """Analyze a failed round's log via ``LogAnalysisBot`` for training feedback. + + Parameters + ---------- + outcome : EvalOutcome + The failed outcome to analyze. + repo_path : str + Absolute path to the repo the task was evaluated against. + model : str + The model to use, in the format ``/``. + log_path : str + Path to the round's log file (the same path passed to + ``run``), analyzed by ``LogAnalysisBot``. + + Returns + ------- + str + Feedback text describing the root cause of the failure and + what the agent's memory notes should cover next time. + """ + bot = LogAnalysisBot(model=model, folder_to_mount=repo_path) + result: BotRunResult = bot.run( + file_name=log_path, + user_prompt=( + "This log was produced while verifying whether an " + "agent completed its task correctly. Identify " + "the root cause of the failure and describe concretely " + "what the agent's memory notes should cover next time to " + "avoid this failure." + ), + ) + + if result.status and result.result: + return result.result + + logger.warning( + "LogAnalysisBot failed to analyze failure (%s); falling back to plain feedback", + result.error, + ) + return ( + f"Evaluation failed. Agent output: {outcome.output}\n" + f"Callback reason: {outcome.result.reason}" + ) + def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> EvalOutcome: """Run one eval iteration: setup -> build_prompt -> WritingBot -> check -> teardown. @@ -385,7 +389,7 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva ------- EvalOutcome The result of this eval round, including the agent's output, - the check verdict, and the round's log file path. + the check verdict. """ self.setup(repo_path) Path(log_path).parent.mkdir(parents=True, exist_ok=True) @@ -416,7 +420,6 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva passed=result.passed, output=bot_result.result, result=result, - log_path=log_path, ) except Exception as exc: logger.exception( @@ -430,7 +433,6 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva result=CallbackResult( passed=False, reason=f"{type(exc).__name__}: {exc}" ), - log_path=log_path, ) finally: try: diff --git a/src/microbots/auto_memory/evalTask.py b/src/microbots/auto_memory/evalTask.py index 6a8925c..d58476a 100644 --- a/src/microbots/auto_memory/evalTask.py +++ b/src/microbots/auto_memory/evalTask.py @@ -35,15 +35,11 @@ class EvalOutcome: The agent's raw output for the round, if any. result : CallbackResult The verdict produced by ``EvalTask.check``. - log_path : str - Path to the round's log file, containing the agent output and - any failure/exception details recorded during the round. """ passed: bool output: str | None result: CallbackResult - log_path: str class EvalTask(ABC): @@ -152,6 +148,35 @@ def teardown(self, repo_path: str) -> None: """ pass + @abstractmethod + def build_feedback(self, outcome: EvalOutcome, repo_path: str, model: str, log_path: str) -> str: + """Required. Analyze a failed eval outcome and produce training feedback. + + Called by the orchestrator after a failed round, before + retraining, to turn the round's outcome/log into concrete + feedback text describing what went wrong and what the agent's + memory notes should cover next time. + + Parameters + ---------- + outcome : EvalOutcome + The failed outcome to analyze. + repo_path : str + Absolute path to the repo the task was evaluated against. + model : str + The model to use, in the format ``/``. + log_path : str + Path to the round's log file, containing the agent output + and any failure/exception details recorded during the + round (the same path passed to ``run``). + + Returns + ------- + str + Feedback text to pass as ``feedback`` to the next round's + training. + """ + @abstractmethod def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> EvalOutcome: """Required. Run one eval iteration and return its outcome. @@ -174,5 +199,5 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva ------- EvalOutcome The result of this eval round, including the agent's output, - the check verdict, and the round's log file path. + the check verdict. """ diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index 0c29538..69b36dc 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -11,7 +11,6 @@ import json import subprocess -from microbots.auto_memory.analyzer import build_feedback from microbots.auto_memory.evalTask import EvalOutcome, EvalTask from microbots.auto_memory.training.runner import run_training from microbots.auto_memory.workdir import ( @@ -197,9 +196,8 @@ def run_train_eval_loop( "run_train_eval_loop: round %d/%d starting", round_idx, max_rounds ) memory_dir = str(load_round_memory(workdir, round_idx, instance_id=task.task_id)) - outcome = task.run( - repo_path, memory_dir, model, str(eval_log_path(workdir, round_idx, task.task_id)) - ) + log_path = str(eval_log_path(workdir, round_idx, task.task_id)) + outcome = task.run(repo_path, memory_dir, model, log_path) outcomes.append(outcome) try: @@ -220,7 +218,7 @@ def run_train_eval_loop( outcome.result.reason, ) try: - feedback = build_feedback(task, outcome, repo_path, model) + feedback = task.build_feedback(outcome, repo_path, model, log_path) run_training_loop( repo_path=repo_path, feedback=feedback, diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py index 771066c..bcaf616 100644 --- a/src/microbots/auto_memory/workdir.py +++ b/src/microbots/auto_memory/workdir.py @@ -275,27 +275,6 @@ def round_log_path(workdir: Path, round_num: int, *, instance_id: str | None = N return round_dir(workdir, round_num, instance_id=instance_id) / ROUND_LOG_FILENAME -def round_patch_path(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: - """Return the path to a round's captured repo diff. - - Parameters - ---------- - workdir : Path - The run's workdir. - round_num : int - 1-based round number. - instance_id : str | None - The eval task's ``task_id``, if running an eval task (see - ``round_dir``). Omit for training-only mode. - - Returns - ------- - Path - This round's ``repo.patch``. - """ - return round_dir(workdir, round_num, instance_id=instance_id) / ROUND_PATCH_FILENAME - - def eval_dir( workdir: Path, round_num: int, instance_id: str, *, create: bool = False ) -> Path: diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py index 98a89cc..56e1906 100644 --- a/test/auto_memory/eval/test_swebenchverified.py +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -18,6 +18,7 @@ load_instances_of_repo, ) from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome +from microbots.MicroBot import BotRunResult MODULE = "microbots.auto_memory.eval.swebenchverified" @@ -156,7 +157,6 @@ def test_build_result_includes_dataset_fields(): passed=True, output="agent output", result=CallbackResult(passed=True, reason="resolved"), - log_path="/dev/null", ) assert task.build_result(outcome) == { @@ -177,6 +177,85 @@ def test_teardown_removes_repo_path(mock_run): mock_run.assert_called_once_with(["rm", "-rf", "/repo"], check=False) +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.build_feedback +# --------------------------------------------------------------------------- + +def _failed_outcome(reason: str = "tests failed", output: str = "agent output") -> EvalOutcome: + return EvalOutcome( + passed=False, + output=output, + result=CallbackResult(passed=False, reason=reason), + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.LogAnalysisBot") +def test_build_feedback_returns_bot_result_on_success(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult( + status=True, result="root cause: missing edge case handling", error=None + ) + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + outcome = _failed_outcome() + + feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log") + + assert feedback == "root cause: missing edge case handling" + mock_bot_cls.assert_called_once_with(model="azure-openai/gpt-4o", folder_to_mount="/repo") + mock_bot.run.assert_called_once() + assert mock_bot.run.call_args.kwargs["file_name"] == "/tmp/some.log" + + +@pytest.mark.unit +@patch(f"{MODULE}.LogAnalysisBot") +def test_build_feedback_falls_back_when_bot_status_false(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed") + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + outcome = _failed_outcome(reason="tests failed", output="some output") + + feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log") + + assert "some output" in feedback + assert "tests failed" in feedback + + +@pytest.mark.unit +@patch(f"{MODULE}.LogAnalysisBot") +def test_build_feedback_falls_back_when_result_is_empty(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="", error=None) + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + outcome = _failed_outcome(reason="assertion error", output="agent tried X") + + feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log") + + assert "agent tried X" in feedback + assert "assertion error" in feedback + + +@pytest.mark.unit +@patch(f"{MODULE}.LogAnalysisBot") +def test_build_feedback_falls_back_when_result_is_none(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result=None, error=None) + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + outcome = _failed_outcome() + + feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log") + + assert "Evaluation failed" in feedback + + # --------------------------------------------------------------------------- # SweBenchVerifiedTask.check # --------------------------------------------------------------------------- @@ -319,7 +398,7 @@ def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert calls[0] == ("setup", "/repo") - assert calls[1] == ("check", "/repo", "agent did stuff", outcome.log_path) + assert calls[1] == ("check", "/repo", "agent did stuff", str(tmp_path / "eval.log")) assert calls[2] == ("teardown", "/repo") assert outcome.passed is True assert outcome.output == "agent did stuff" @@ -393,7 +472,7 @@ def _build_prompt(): assert outcome.passed is False assert "bad prompt" in outcome.result.reason - with open(outcome.log_path) as f: + with open(str(tmp_path / "eval.log")) as f: assert "bad prompt" in f.read() @@ -467,63 +546,6 @@ def _teardown(repo_path): -# --------------------------------------------------------------------------- -# SweBenchVerifiedTask.add_cli_args / from_cli_args -# --------------------------------------------------------------------------- - -@pytest.mark.unit -def test_add_cli_args_registers_instance_id_and_repo_flags(): - import argparse - - parser = argparse.ArgumentParser() - SweBenchVerifiedTask.add_cli_args(parser) - - args = parser.parse_args(["--instance-id", "django__django-1", "--swebench-repo", "django/django"]) - assert args.instance_id == "django__django-1" - assert args.swebench_repo == "django/django" - - -@pytest.mark.unit -@patch(f"{MODULE}.load_instance_using_id") -def test_from_cli_args_uses_instance_id_when_given(mock_load_instance_using_id): - mock_load_instance_using_id.return_value = _instance() - args = MagicMock(instance_id="django__django-1", swebench_repo=None) - - tasks = SweBenchVerifiedTask.from_cli_args(args) - - mock_load_instance_using_id.assert_called_once_with("django__django-1") - assert len(tasks) == 1 - assert isinstance(tasks[0], SweBenchVerifiedTask) - assert tasks[0].instance == _instance() - - -@pytest.mark.unit -@patch(f"{MODULE}.load_instances_of_repo") -def test_from_cli_args_falls_back_to_repo_filter_when_no_instance_id(mock_load_instances_of_repo): - mock_load_instances_of_repo.return_value = [_instance(), _instance()] - args = MagicMock(instance_id=None, swebench_repo="django/django") - - tasks = SweBenchVerifiedTask.from_cli_args(args) - - mock_load_instances_of_repo.assert_called_once_with(repo="django/django") - assert len(tasks) == 2 - assert all(isinstance(t, SweBenchVerifiedTask) for t in tasks) - - -@pytest.mark.unit -@patch(f"{MODULE}.load_instances_of_repo") -def test_from_cli_args_handles_missing_attrs_gracefully(mock_load_instances_of_repo): - """Namespace without instance_id/swebench_repo attrs at all (not just None).""" - mock_load_instances_of_repo.return_value = [_instance()] - - class _EmptyArgs: - pass - - tasks = SweBenchVerifiedTask.from_cli_args(_EmptyArgs()) - - mock_load_instances_of_repo.assert_called_once_with(repo=None) - assert len(tasks) == 1 - # --------------------------------------------------------------------------- # SweBenchVerifiedTask.from_config diff --git a/test/auto_memory/test_analyzer.py b/test/auto_memory/test_analyzer.py deleted file mode 100644 index 3566b65..0000000 --- a/test/auto_memory/test_analyzer.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Unit tests for microbots.auto_memory.analyzer.""" - -import os -import sys -from unittest.mock import MagicMock, patch - -import pytest - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) - -from microbots.auto_memory.analyzer import build_feedback -from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome -from microbots.MicroBot import BotRunResult - - -def _make_outcome(reason: str = "tests failed", output: str = "agent output") -> EvalOutcome: - return EvalOutcome( - passed=False, - output=output, - result=CallbackResult(passed=False, reason=reason), - log_path="/tmp/some.log", - ) - - -@pytest.mark.unit -@patch("microbots.auto_memory.analyzer.LogAnalysisBot") -def test_build_feedback_returns_bot_result_on_success(mock_bot_cls): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult( - status=True, result="root cause: missing edge case handling", error=None - ) - mock_bot_cls.return_value = mock_bot - - outcome = _make_outcome() - feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") - - assert feedback == "root cause: missing edge case handling" - mock_bot_cls.assert_called_once_with(model="azure-openai/gpt-4o", folder_to_mount="/repo") - mock_bot.run.assert_called_once() - assert mock_bot.run.call_args.kwargs["file_name"] == outcome.log_path - - -@pytest.mark.unit -@patch("microbots.auto_memory.analyzer.LogAnalysisBot") -def test_build_feedback_falls_back_when_bot_status_false(mock_bot_cls): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed") - mock_bot_cls.return_value = mock_bot - - outcome = _make_outcome(reason="tests failed", output="some output") - feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") - - assert "some output" in feedback - assert "tests failed" in feedback - - -@pytest.mark.unit -@patch("microbots.auto_memory.analyzer.LogAnalysisBot") -def test_build_feedback_falls_back_when_result_is_empty(mock_bot_cls): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="", error=None) - mock_bot_cls.return_value = mock_bot - - outcome = _make_outcome(reason="assertion error", output="agent tried X") - feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") - - assert "agent tried X" in feedback - assert "assertion error" in feedback - - -@pytest.mark.unit -@patch("microbots.auto_memory.analyzer.LogAnalysisBot") -def test_build_feedback_falls_back_when_result_is_none(mock_bot_cls): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result=None, error=None) - mock_bot_cls.return_value = mock_bot - - outcome = _make_outcome() - feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") - - assert "Evaluation failed" in feedback diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py index 10867f2..fba3829 100644 --- a/test/auto_memory/test_orchestrator.py +++ b/test/auto_memory/test_orchestrator.py @@ -25,12 +25,11 @@ MODULE = "microbots.auto_memory.orchestrator" -def _make_outcome(passed: bool, log_path: str, reason: str = "reason") -> EvalOutcome: +def _make_outcome(passed: bool, reason: str = "reason") -> EvalOutcome: return EvalOutcome( passed=passed, output="agent output", result=CallbackResult(passed=passed, reason=reason), - log_path=log_path, ) @@ -52,11 +51,9 @@ def _make_task() -> MagicMock: @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, mock_run_training_loop, tmp_path): - log_path = _touch(str(tmp_path / "round1.log")) +def test_loop_returns_immediately_when_first_round_passes(mock_run_training_loop, tmp_path): task = _make_task() - task.run.return_value = _make_outcome(passed=True, log_path=log_path) + task.run.return_value = _make_outcome(passed=True) result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) @@ -64,28 +61,25 @@ def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, m assert result.passed is True assert result.rounds_run == 1 assert task.run.call_count == 1 - mock_build_feedback.assert_not_called() + task.build_feedback.assert_not_called() mock_run_training_loop.assert_not_called() @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, mock_run_training_loop, tmp_path): - log1 = _touch(str(tmp_path / "round1.log")) - log2 = _touch(str(tmp_path / "round2.log")) +def test_loop_retrains_and_continues_on_failure_then_passes(mock_run_training_loop, tmp_path): task = _make_task() task.run.side_effect = [ - _make_outcome(passed=False, log_path=log1), - _make_outcome(passed=True, log_path=log2), + _make_outcome(passed=False), + _make_outcome(passed=True), ] - mock_build_feedback.return_value = "feedback text" + task.build_feedback.return_value = "feedback text" result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert result.passed is True assert result.rounds_run == 2 - mock_build_feedback.assert_called_once() + task.build_feedback.assert_called_once() mock_run_training_loop.assert_called_once_with( repo_path="/repo", feedback="feedback text", @@ -97,14 +91,13 @@ def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_training_loop, tmp_path): +def test_loop_exhausts_max_rounds_without_passing(mock_run_training_loop, tmp_path): task = _make_task() task.run.side_effect = [ - _make_outcome(passed=False, log_path=_touch(str(tmp_path / f"round{i}.log"))) + _make_outcome(passed=False) for i in range(3) ] - mock_build_feedback.return_value = "feedback text" + task.build_feedback.return_value = "feedback text" result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=3) @@ -112,17 +105,16 @@ def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_ assert result.rounds_run == 3 assert len(result.outcomes) == 3 assert result.final_outcome is result.outcomes[-1] - assert mock_build_feedback.call_count == 3 + assert task.build_feedback.call_count == 3 assert mock_run_training_loop.call_count == 3 @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_log_path_persists_after_passing_round(mock_build_feedback, mock_run_training_loop, tmp_path): +def test_log_path_persists_after_passing_round(mock_run_training_loop, tmp_path): log_path = _touch(str(tmp_path / "round1.log")) task = _make_task() - task.run.return_value = _make_outcome(passed=True, log_path=log_path) + task.run.return_value = _make_outcome(passed=True) run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) @@ -131,16 +123,15 @@ def test_log_path_persists_after_passing_round(mock_build_feedback, mock_run_tra @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_log_path_persists_after_failing_round(mock_build_feedback, mock_run_training_loop, tmp_path): +def test_log_path_persists_after_failing_round(mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) task = _make_task() task.run.side_effect = [ - _make_outcome(passed=False, log_path=log1), - _make_outcome(passed=True, log_path=log2), + _make_outcome(passed=False), + _make_outcome(passed=True), ] - mock_build_feedback.return_value = "feedback text" + task.build_feedback.return_value = "feedback text" run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) @@ -150,16 +141,14 @@ def test_log_path_persists_after_failing_round(mock_build_feedback, mock_run_tra @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_run_training_loop, tmp_path): +def test_build_feedback_exception_does_not_crash_loop(mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) - log2 = _touch(str(tmp_path / "round2.log")) task = _make_task() task.run.side_effect = [ - _make_outcome(passed=False, log_path=log1), - _make_outcome(passed=True, log_path=log2), + _make_outcome(passed=False), + _make_outcome(passed=True), ] - mock_build_feedback.side_effect = RuntimeError("analysis bot crashed") + task.build_feedback.side_effect = RuntimeError("analysis bot crashed") result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) @@ -171,16 +160,13 @@ def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_ @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_loop_forwards_training_iterations_to_run_training_loop(mock_build_feedback, mock_run_training_loop, tmp_path): - log1 = _touch(str(tmp_path / "round1.log")) - log2 = _touch(str(tmp_path / "round2.log")) +def test_loop_forwards_training_iterations_to_run_training_loop(mock_run_training_loop, tmp_path): task = _make_task() task.run.side_effect = [ - _make_outcome(passed=False, log_path=log1), - _make_outcome(passed=True, log_path=log2), + _make_outcome(passed=False), + _make_outcome(passed=True), ] - mock_build_feedback.return_value = "feedback text" + task.build_feedback.return_value = "feedback text" run_train_eval_loop( "/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5, training_iterations=4 @@ -197,16 +183,14 @@ def test_loop_forwards_training_iterations_to_run_training_loop(mock_build_feedb @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_run_training_loop, tmp_path): +def test_run_training_exception_does_not_crash_loop(mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) - log2 = _touch(str(tmp_path / "round2.log")) task = _make_task() task.run.side_effect = [ - _make_outcome(passed=False, log_path=log1), - _make_outcome(passed=True, log_path=log2), + _make_outcome(passed=False), + _make_outcome(passed=True), ] - mock_build_feedback.return_value = "feedback text" + task.build_feedback.return_value = "feedback text" mock_run_training_loop.side_effect = RuntimeError("training crashed") result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) @@ -292,7 +276,7 @@ def test_write_eval_result_writes_task_build_result_as_json(tmp_path): task = MagicMock() task.task_id = "django__django-1" task.build_result.return_value = {"passed": True, "reason": "resolved"} - outcome = _make_outcome(passed=True, log_path="/dev/null") + outcome = _make_outcome(passed=True) write_eval_result(tmp_path, 2, task, outcome) @@ -306,7 +290,7 @@ def test_write_eval_result_creates_missing_parent_dirs(tmp_path): task = MagicMock() task.task_id = "some-task" task.build_result.return_value = {"passed": False, "reason": "nope"} - outcome = _make_outcome(passed=False, log_path="/dev/null") + outcome = _make_outcome(passed=False) write_eval_result(tmp_path, 1, task, outcome) @@ -314,16 +298,13 @@ def test_write_eval_result_creates_missing_parent_dirs(tmp_path): @pytest.mark.unit -@patch(f"{MODULE}.build_feedback") -def test_loop_writes_eval_result_for_every_round(mock_build_feedback, tmp_path): - log1 = _touch(str(tmp_path / "round1.log")) - log2 = _touch(str(tmp_path / "round2.log")) +def test_loop_writes_eval_result_for_every_round(tmp_path): task = _make_task() task.run.side_effect = [ - _make_outcome(passed=False, log_path=log1), - _make_outcome(passed=True, log_path=log2), + _make_outcome(passed=False), + _make_outcome(passed=True), ] - mock_build_feedback.return_value = "feedback text" + task.build_feedback.return_value = "feedback text" with patch(f"{MODULE}.run_training_loop"): run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) @@ -428,12 +409,8 @@ def fake_train(repo_path, feedback, memory_dir, model, iterations=1): @pytest.mark.unit -@patch(f"{MODULE}.build_feedback") @patch(f"{MODULE}.run_training_loop") -def test_loop_carries_memory_forward_between_rounds(mock_run_training_loop, mock_build_feedback, tmp_path): - log1 = _touch(str(tmp_path / "round1.log")) - log2 = _touch(str(tmp_path / "round2.log")) - mock_build_feedback.return_value = "feedback text" +def test_loop_carries_memory_forward_between_rounds(mock_run_training_loop, tmp_path): seen_memory_dirs = [] def fake_run(repo_path, memory_dir, model, log_path): @@ -443,11 +420,11 @@ def fake_run(repo_path, memory_dir, model, log_path): assert (Path(memory_dir) / "notes.md").read_text() == "round 1 progress" seen_memory_dirs.append(memory_dir) Path(memory_dir, "notes.md").write_text(f"round {round_num} progress") - log_path = log1 if round_num == 1 else log2 - return _make_outcome(passed=round_num == 2, log_path=log_path) + return _make_outcome(passed=round_num == 2) task = _make_task() task.run.side_effect = fake_run + task.build_feedback.return_value = "feedback text" run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) diff --git a/test/auto_memory/test_task.py b/test/auto_memory/test_task.py index ed11bb2..b66deb1 100644 --- a/test/auto_memory/test_task.py +++ b/test/auto_memory/test_task.py @@ -11,16 +11,18 @@ class _RunOnlyTask(EvalTask): - """A task that overrides only run(), never touching the optional hooks.""" + """A task that overrides only run()/build_feedback(), never touching the optional hooks.""" def run(self, repo_path, memory_dir, model, log_path): return EvalOutcome( passed=True, output="custom output", result=None, - log_path="/dev/null", ) + def build_feedback(self, outcome, repo_path, model, log_path): + return "feedback text" + @pytest.mark.unit def test_run_is_abstract(): @@ -28,6 +30,16 @@ def test_run_is_abstract(): EvalTask() +@pytest.mark.unit +def test_build_feedback_is_abstract(): + class _MissingBuildFeedback(EvalTask): + def run(self, repo_path, memory_dir, model, log_path): + raise NotImplementedError + + with pytest.raises(TypeError): + _MissingBuildFeedback() + + @pytest.mark.unit def test_subclass_overriding_only_run_is_instantiable(): task = _RunOnlyTask() @@ -73,7 +85,6 @@ def test_default_build_result_returns_passed_and_reason(): passed=False, output="agent output", result=CallbackResult(passed=False, reason="check failed"), - log_path="/dev/null", ) assert _RunOnlyTask().build_result(outcome) == {"passed": False, "reason": "check failed"} diff --git a/test/auto_memory/test_task_registry.py b/test/auto_memory/test_task_registry.py index 4caa88f..ea34cf8 100644 --- a/test/auto_memory/test_task_registry.py +++ b/test/auto_memory/test_task_registry.py @@ -31,6 +31,9 @@ def check(self, output): def teardown(self, repo_path): pass + def build_feedback(self, outcome, repo_path, model, log_path): + return "feedback" + def run(self, repo_path, memory_dir, model): return super().run(repo_path, memory_dir, model) diff --git a/test/auto_memory/test_workdir.py b/test/auto_memory/test_workdir.py index 149b223..1eb2fae 100644 --- a/test/auto_memory/test_workdir.py +++ b/test/auto_memory/test_workdir.py @@ -22,7 +22,6 @@ round_dir, round_log_path, round_memory_dir, - round_patch_path, run_log_path, save_round_memory, ) @@ -175,18 +174,6 @@ def test_round_log_path_with_instance_id(tmp_path): ) / "round.log" -@pytest.mark.unit -def test_round_patch_path_returns_repo_patch(tmp_path): - assert round_patch_path(tmp_path, 2) == round_dir(tmp_path, 2) / "repo.patch" - - -@pytest.mark.unit -def test_round_patch_path_with_instance_id(tmp_path): - assert round_patch_path(tmp_path, 2, instance_id="task-1") == round_dir( - tmp_path, 2, instance_id="task-1" - ) / "repo.patch" - - @pytest.mark.unit def test_eval_dir_creates_directory_when_requested(tmp_path): path = eval_dir(tmp_path, 1, "task-1", create=True) From 021c7efe80cbf99c55431916d8ebf7d4d286224f Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Thu, 3 Sep 2026 07:31:11 +0000 Subject: [PATCH 11/21] Refactor --- .../auto_memory/eval/swebenchverified.py | 198 ++++++++++-------- src/microbots/auto_memory/evalTask.py | 40 +++- src/microbots/auto_memory/orchestrator.py | 49 ++--- src/microbots/auto_memory/workdir.py | 60 +++++- .../auto_memory/eval/test_swebenchverified.py | 186 +++++++++------- test/auto_memory/test_orchestrator.py | 55 ++--- test/auto_memory/test_task.py | 27 +++ test/auto_memory/test_task_registry.py | 4 + test/auto_memory/test_workdir.py | 42 +++- 9 files changed, 440 insertions(+), 221 deletions(-) diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index 2fd64b1..889693b 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -54,32 +54,10 @@ def _load_dataset_rows(dataset_name: str): return load_dataset(dataset_name, split="test") -@dataclass -class SweBenchInstance: - """A single SWE-bench-verified dataset row. - - Attributes - ---------- - instance_id : str - Unique identifier for the instance, e.g. ``"django__django-11099"``. - repo : str - The GitHub repo this instance belongs to, e.g. ``"django/django"``. - base_commit : str - Commit hash representing the repo state before the issue's fix. - problem_statement : str - The GitHub issue title and body describing the bug to fix. - """ - - instance_id: str - repo: str - base_commit: str - problem_statement: str - - def load_instances_of_repo( dataset_name: str = SWE_BENCH_VERIFIED, repo: str | None = None, -) -> list[SweBenchInstance]: +) -> list["SweBenchInstance"]: """Load all dataset instances, optionally filtered to a single repo. Parameters @@ -110,7 +88,7 @@ def load_instances_of_repo( ] return instances -def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIFIED) -> SweBenchInstance: +def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIFIED) -> "SweBenchInstance": """Load a single dataset instance by its instance ID. Parameters @@ -143,6 +121,27 @@ def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIF ) raise ValueError(f"instance_id not found: {instance_id}") +@dataclass +class SweBenchInstance: + """A single SWE-bench-verified dataset row. + + Attributes + ---------- + instance_id : str + Unique identifier for the instance, e.g. ``"django__django-11099"``. + repo : str + The GitHub repo this instance belongs to, e.g. ``"django/django"``. + base_commit : str + Commit hash representing the repo state before the issue's fix. + problem_statement : str + The GitHub issue title and body describing the bug to fix. + """ + + instance_id: str + repo: str + base_commit: str + problem_statement: str + @register_task("swebenchverified") class SweBenchVerifiedTask(EvalTask): """Eval task that verifies a fix against one SWE-bench-verified instance. @@ -225,17 +224,48 @@ def build_result(self, outcome: EvalOutcome) -> dict: } def setup(self, repo_path: str) -> None: - """Clone the instance's repo and check out its base commit. + """Clone the instance's repo, or reset it, to its base commit. + + Clones fresh on first use. If ``repo_path`` already exists + (e.g. left behind by a previous round) *and* its ``origin`` + remote matches this instance's repo, it's reset instead of + re-cloned: ``git reset --hard `` followed by + ``git clean -fd`` discards whatever the agent changed, without + the cost of a full re-clone and without deleting the directory + ``build_feedback`` may still need to inspect afterward. If + ``repo_path`` exists but isn't a checkout of this repo (e.g. a + stale directory left over from a different run/task), it's + removed and cloned fresh instead, to avoid silently operating + on the wrong codebase. Parameters ---------- repo_path : str - Absolute path to clone the repo into. + Absolute path to clone (or reset) the repo into. """ - subprocess.run( - ["git", "clone", f"https://github.com/{self.instance.repo}.git", repo_path], - check=True, - ) + expected_url = f"https://github.com/{self.instance.repo}.git" + + if Path(repo_path).exists(): + origin = subprocess.run( + ["git", "remote", "get-url", "origin"], + cwd=repo_path, capture_output=True, text=True, + ) + if origin.returncode == 0 and origin.stdout.strip() == expected_url: + subprocess.run( + ["git", "reset", "--hard", self.instance.base_commit], + cwd=repo_path, check=True, + ) + subprocess.run(["git", "clean", "-fd"], cwd=repo_path, check=True) + return + + logger.warning( + "SweBenchVerifiedTask.setup: %s exists but isn't a checkout of %s " + "(origin=%r); removing and re-cloning", + repo_path, expected_url, origin.stdout.strip(), + ) + shutil.rmtree(repo_path) + + subprocess.run(["git", "clone", expected_url, repo_path], check=True) subprocess.run( ["git", "checkout", self.instance.base_commit], cwd=repo_path, check=True ) @@ -266,7 +296,10 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes verification is based on the repo's git diff, not the agent's textual output. log_path : str - Path to a log file to append the harness's output to. + Path to a log file to append the harness's output to, + including the per-instance ``run_instance.log`` and + ``test_output.txt`` artifacts if the harness produced them + (read before the harness's ``report_dir`` is cleaned up). Returns ------- @@ -280,6 +313,8 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes run_id = f"microbots-{uuid.uuid4().hex[:8]}" model_name_or_path = EVAL_AGENT_MODEL_NAME pred_path = Path(tempfile.mktemp(suffix=".json")) + #will need to update when upgraded to ~5.0.2 , removed this flag in the new version + #https://github.com/SWE-bench/SWE-bench/commit/e2c13307b6cf7764a50958b9c8bfbfb3f72cb70a report_dir = Path(tempfile.mkdtemp()) pred_path.write_text(json.dumps([{ "instance_id": self.instance.instance_id, @@ -300,30 +335,29 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes capture_output=True, text=True, cwd=report_dir, ) + #need to update this instance_log_dir path when swebench is upgraded + instance_log_dir = ( + report_dir / "logs" / "run_evaluation" / run_id + / model_name_or_path / self.instance.instance_id + ) with open(log_path, "a") as f: f.write(proc.stdout + proc.stderr) + for log_filename in ("run_instance.log", "test_output.txt"): + log_file = instance_log_dir / log_filename + if log_file.exists(): + f.write(f"\n--- {log_filename} ---\n{log_file.read_text()}\n") - report_file = report_dir / f"{model_name_or_path}.{run_id}.json" + report_file = instance_log_dir / "report.json" passed = False if report_file.exists(): report = json.loads(report_file.read_text()) - passed = self.instance.instance_id in report.get("resolved_ids", []) + passed = report.get(self.instance.instance_id, {}).get("resolved", False) finally: pred_path.unlink(missing_ok=True) shutil.rmtree(report_dir, ignore_errors=True) return CallbackResult(passed=passed, reason="resolved" if passed else "not resolved") - def teardown(self, repo_path: str) -> None: - """Remove the cloned repo working directory. - - Parameters - ---------- - repo_path : str - Absolute path to the repo cloned by ``setup``. - """ - subprocess.run(["rm", "-rf", repo_path], check=False) - def build_feedback(self, outcome: EvalOutcome, repo_path: str, model: str, log_path: str) -> str: """Analyze a failed round's log via ``LogAnalysisBot`` for training feedback. @@ -370,7 +404,7 @@ def build_feedback(self, outcome: EvalOutcome, repo_path: str, model: str, log_p ) def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> EvalOutcome: - """Run one eval iteration: setup -> build_prompt -> WritingBot -> check -> teardown. + """Run one eval iteration: setup -> build_prompt -> WritingBot -> check. Parameters ---------- @@ -396,48 +430,42 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva Path(log_path).write_text("") try: - try: - prompt = self.build_prompt() - bot = WritingBot( - model=model, - folder_to_mount=repo_path, - additional_tools=[MemoryTool(memory_dir=memory_dir)], - ) - bot_result = bot.run(prompt) + prompt = self.build_prompt() + bot = WritingBot( + model=model, + folder_to_mount=repo_path, + additional_tools=[MemoryTool(memory_dir=memory_dir)], + ) + bot_result = bot.run(prompt) + with open(log_path, "a") as f: + f.write(f"Agent output:\n{bot_result.result}\n") + + if not bot_result.status: + reason = f"Bot run failed: {bot_result.error}" with open(log_path, "a") as f: - f.write(f"Agent output:\n{bot_result.result}\n") - - if not bot_result.status: - reason = f"Bot run failed: {bot_result.error}" - with open(log_path, "a") as f: - f.write(f"\n{reason}\n") - result = CallbackResult(passed=False, reason=reason) - else: - result = self.check(repo_path, bot_result.result or "", log_path) - - return EvalOutcome( - passed=result.passed, - output=bot_result.result, - result=result, - ) - except Exception as exc: - logger.exception( - "SweBenchVerifiedTask.run: iteration raised %s", type(exc).__name__ - ) - with open(log_path, "a") as f: - f.write(f"\nException during eval iteration: {type(exc).__name__}: {exc}\n") - return EvalOutcome( - passed=False, - output=None, - result=CallbackResult( - passed=False, reason=f"{type(exc).__name__}: {exc}" - ), - ) - finally: - try: - self.teardown(repo_path) - except Exception: - logger.exception("SweBenchVerifiedTask.run: teardown() raised exception; ignoring") + f.write(f"\n{reason}\n") + result = CallbackResult(passed=False, reason=reason) + else: + result = self.check(repo_path, bot_result.result or "", log_path) + + return EvalOutcome( + passed=result.passed, + output=bot_result.result, + result=result, + ) + except Exception as exc: + logger.exception( + "SweBenchVerifiedTask.run: iteration raised %s", type(exc).__name__ + ) + with open(log_path, "a") as f: + f.write(f"\nException during eval iteration: {type(exc).__name__}: {exc}\n") + return EvalOutcome( + passed=False, + output=None, + result=CallbackResult( + passed=False, reason=f"{type(exc).__name__}: {exc}" + ), + ) diff --git a/src/microbots/auto_memory/evalTask.py b/src/microbots/auto_memory/evalTask.py index d58476a..aacee20 100644 --- a/src/microbots/auto_memory/evalTask.py +++ b/src/microbots/auto_memory/evalTask.py @@ -7,6 +7,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass +from typing import Any @dataclass class CallbackResult: @@ -45,11 +46,11 @@ class EvalOutcome: class EvalTask(ABC): """Base class for a single evaluation task in the train <-> eval loop. - Subclasses must implement ``run``. ``setup``, ``build_prompt``, - ``check``, and ``teardown`` are optional hooks subclasses may use - to structure their own ``run`` implementation (see - ``SweBenchVerifiedTask`` for an example), but nothing in this base - class calls them automatically. + Subclasses must implement ``run`` and ``from_config``. ``setup``, + ``build_prompt``, ``check``, and ``teardown`` are optional hooks + subclasses may use to structure their own ``run`` implementation + (see ``SweBenchVerifiedTask`` for an example), but nothing in this + base class calls them automatically. """ @property @@ -148,6 +149,35 @@ def teardown(self, repo_path: str) -> None: """ pass + @classmethod + @abstractmethod + def from_config(cls, task_args: dict[str, Any]) -> list["EvalTask"]: + """Required. Build task instance(s) from a config's ``task_args`` dict. + + Called by the CLI at runtime (driven by ``--task``) to + construct the actual task object(s) to run, using whatever + config values the task needs (e.g. a dataset instance ID, a + repo filter). Object creation must go through this method + rather than being constructed elsewhere, so behavior stays + driven by the CLI/config at runtime. + + Parameters + ---------- + task_args : dict[str, Any] + Task-specific config values (the config file's + ``task_args`` section). + + Returns + ------- + list[EvalTask] + One task instance per unit of work this config describes + (often just one, but e.g. ``SweBenchVerifiedTask`` returns + one per matching dataset instance). + """ + raise NotImplementedError( + f"{cls.__name__} must implement from_config() to be usable via --task" + ) + @abstractmethod def build_feedback(self, outcome: EvalOutcome, repo_path: str, model: str, log_path: str) -> str: """Required. Analyze a failed eval outcome and produce training feedback. diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index 69b36dc..7dc5662 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -15,11 +15,13 @@ from microbots.auto_memory.training.runner import run_training from microbots.auto_memory.workdir import ( eval_log_path, + eval_repo_dir, eval_result_path, load_config, load_round_memory, repo_dir, save_round_memory, + snapshot_seed_memory, ) logger = getLogger(__name__) @@ -60,24 +62,6 @@ def clone_repo(url: str, repo_path: Path) -> None: return subprocess.run(["git", "clone", url, str(repo_path)], check=True) -def reset_repo(repo_path: Path, base_commit: str) -> None: - """Reset ``repo_path`` to ``base_commit``, discarding all local changes. - - Runs ``git reset --hard `` followed by ``git clean -fd``, - so every round/instance starts from the same pristine state instead - of carrying forward whatever a previous round or eval attempt left - behind. - - Parameters - ---------- - repo_path : Path - Path to the repo to reset. - base_commit : str - Commit-ish to reset to. - """ - subprocess.run(["git", "reset", "--hard", base_commit], cwd=repo_path, check=True) - subprocess.run(["git", "clean", "-fd"], cwd=repo_path, check=True) - def write_eval_result(workdir: Path, round_num: int, task: EvalTask, outcome: EvalOutcome) -> None: """Write a round's eval result to ``result.json``. @@ -141,7 +125,8 @@ def run_training_loop( ) def run_train_eval_loop( - repo_path: str, + training_repo_path: str, + eval_repo_path: str, workdir: Path, model: str, task: EvalTask, @@ -168,8 +153,15 @@ def run_train_eval_loop( Parameters ---------- - repo_path : str - Absolute path to the repo to evaluate and train against. + training_repo_path : str + Absolute path to the persistent repo checkout used only for + retraining (``run_training_loop``). Kept separate from + ``eval_repo_path`` since the task manages the latter's + lifecycle itself (clone/teardown each round). + eval_repo_path : str + Absolute path to the repo the task clones/manages itself (via + its own ``setup``) and runs/checks the agent against each + round. workdir : Path This run's workdir, used to carry memory forward between rounds (see ``microbots.auto_memory.workdir``). @@ -197,7 +189,7 @@ def run_train_eval_loop( ) memory_dir = str(load_round_memory(workdir, round_idx, instance_id=task.task_id)) log_path = str(eval_log_path(workdir, round_idx, task.task_id)) - outcome = task.run(repo_path, memory_dir, model, log_path) + outcome = task.run(eval_repo_path, memory_dir, model, log_path) outcomes.append(outcome) try: @@ -218,9 +210,9 @@ def run_train_eval_loop( outcome.result.reason, ) try: - feedback = task.build_feedback(outcome, repo_path, model, log_path) + feedback = task.build_feedback(outcome, eval_repo_path, model, log_path) run_training_loop( - repo_path=repo_path, + repo_path=training_repo_path, feedback=feedback, memory_dir=memory_dir, model=model, @@ -290,14 +282,16 @@ def run( if repo_url: clone_repo(repo_url, repo_dir(workdir)) - repo_path = str(repo_dir(workdir)) + snapshot_seed_memory(workdir) + + training_repo_path = str(repo_dir(workdir)) if task is None: # Train-only mode has no rounds of its own; round 1 is just a # scratch dir seeded from (and saved back to) top-level memory. memory_dir = str(load_round_memory(workdir, 1)) run_training_loop( - repo_path=repo_path, + repo_path=training_repo_path, feedback="", memory_dir=memory_dir, model=model, @@ -307,7 +301,8 @@ def run( return None return run_train_eval_loop( - repo_path=repo_path, + training_repo_path=training_repo_path, + eval_repo_path=str(eval_repo_dir(workdir)), workdir=workdir, model=model, task=task, diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py index bcaf616..65cb3ff 100644 --- a/src/microbots/auto_memory/workdir.py +++ b/src/microbots/auto_memory/workdir.py @@ -13,8 +13,9 @@ WORKDIR_NAME = "workdir" CONFIG_FILENAME = "config.yaml" REPO_DIRNAME = "repo" -RUN_LOG_FILENAME = "run.log" +EVAL_REPO_DIRNAME = "eval_repo" MEMORY_DIRNAME = "memory" +MEMORY_SEED_DIRNAME = "memory_seed" ROUNDS_DIRNAME = "rounds" ROUND_LOG_FILENAME = "round.log" ROUND_PATCH_FILENAME = "repo.patch" @@ -96,6 +97,13 @@ def load_config(workdir: Path) -> dict: def repo_dir(workdir: Path) -> Path: """Return the path to the single cloned repo shared across rounds. + Used only for training (both train-only mode and the eval loop's + retrain step): a persistent checkout that stays in place across + rounds. Eval tasks that manage their own repo checkout (e.g. + ``SweBenchVerifiedTask``, which clones a different repo/commit per + dataset instance) use ``eval_repo_dir`` instead, so the two never + collide. + Parameters ---------- workdir : Path @@ -109,8 +117,14 @@ def repo_dir(workdir: Path) -> Path: return workdir / REPO_DIRNAME -def run_log_path(workdir: Path) -> Path: - """Return the path to the top-level orchestrator log. +def eval_repo_dir(workdir: Path) -> Path: + """Return the path to the repo an eval task clones/manages itself. + + Kept separate from ``repo_dir`` (the training repo) because a + task's ``setup`` may clone or reset this directory every round + (e.g. ``SweBenchVerifiedTask`` checks out a different repo/commit + per dataset instance), which would otherwise conflict with the + persistent training checkout at ``repo_dir``. Parameters ---------- @@ -120,9 +134,9 @@ def run_log_path(workdir: Path) -> Path: Returns ------- Path - ``workdir/run.log``. + ``workdir/eval_repo``. """ - return workdir / RUN_LOG_FILENAME + return workdir / EVAL_REPO_DIRNAME def memory_dir(workdir: Path) -> Path: @@ -141,6 +155,42 @@ def memory_dir(workdir: Path) -> Path: return workdir / MEMORY_DIRNAME +def snapshot_seed_memory(workdir: Path) -> Path: + """Snapshot the current top-level memory dir as the run's restorable baseline. + + ``memory_dir`` is shared and mutated in place across every + training/eval round and every eval task instance (so later + instances benefit from what earlier ones learned), which means the + original, pre-run memory is otherwise overwritten and lost with no + way to get back to it. Call this once, before anything trains, + to preserve that original state at ``workdir/memory_seed``. A + no-op if a snapshot already exists, so later calls (e.g. once per + eval task instance in the same run) never clobber the very first + snapshot with already-mutated memory. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/memory_seed``, containing a copy of whatever + ``memory_dir`` held the first time this was called (or empty, + if there was no pre-existing memory). + """ + dst = workdir / MEMORY_SEED_DIRNAME + if dst.exists(): + return dst + src = memory_dir(workdir) + if src.is_dir(): + shutil.copytree(src, dst) + else: + dst.mkdir(parents=True, exist_ok=True) + return dst + + def round_dir( workdir: Path, round_num: int, *, instance_id: str | None = None, create: bool = False ) -> Path: diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py index 56e1906..79477ec 100644 --- a/test/auto_memory/eval/test_swebenchverified.py +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -114,7 +114,7 @@ def test_dataset_rows_are_cached_across_repeated_calls(mock_load_dataset): # --------------------------------------------------------------------------- -# SweBenchVerifiedTask.setup / build_prompt / teardown +# SweBenchVerifiedTask.setup / build_prompt # --------------------------------------------------------------------------- def _instance(): @@ -128,14 +128,75 @@ def _instance(): @pytest.mark.unit @patch(f"{MODULE}.subprocess.run") -def test_setup_clones_and_checks_out_base_commit(mock_run): +def test_setup_clones_and_checks_out_base_commit_when_repo_missing(mock_run, tmp_path): + repo_path = tmp_path / "repo" task = SweBenchVerifiedTask(_instance()) - task.setup("/repo") + task.setup(str(repo_path)) clone_call, checkout_call = mock_run.call_args_list - assert clone_call.args[0] == ["git", "clone", "https://github.com/django/django.git", "/repo"] + assert clone_call.args[0] == [ + "git", "clone", "https://github.com/django/django.git", str(repo_path) + ] + assert checkout_call.args[0] == ["git", "checkout", "abc123"] + assert checkout_call.kwargs["cwd"] == str(repo_path) + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_setup_resets_instead_of_recloning_when_origin_matches(mock_run, tmp_path): + repo_path = tmp_path / "repo" + repo_path.mkdir() + mock_run.return_value = MagicMock( + returncode=0, stdout="https://github.com/django/django.git\n" + ) + task = SweBenchVerifiedTask(_instance()) + task.setup(str(repo_path)) + + origin_call, reset_call, clean_call = mock_run.call_args_list + assert origin_call.args[0] == ["git", "remote", "get-url", "origin"] + assert origin_call.kwargs["cwd"] == str(repo_path) + assert reset_call.args[0] == ["git", "reset", "--hard", "abc123"] + assert reset_call.kwargs["cwd"] == str(repo_path) + assert clean_call.args[0] == ["git", "clean", "-fd"] + assert clean_call.kwargs["cwd"] == str(repo_path) + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_setup_removes_and_reclones_when_origin_mismatched(mock_run, tmp_path): + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "stale_file.txt").write_text("leftover from a different repo") + mock_run.return_value = MagicMock( + returncode=0, stdout="https://github.com/other/repo.git\n" + ) + task = SweBenchVerifiedTask(_instance()) + task.setup(str(repo_path)) + + assert not (repo_path / "stale_file.txt").exists() + origin_call, clone_call, checkout_call = mock_run.call_args_list + assert origin_call.args[0] == ["git", "remote", "get-url", "origin"] + assert clone_call.args[0] == [ + "git", "clone", "https://github.com/django/django.git", str(repo_path) + ] assert checkout_call.args[0] == ["git", "checkout", "abc123"] - assert checkout_call.kwargs["cwd"] == "/repo" + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_setup_removes_and_reclones_when_repo_path_not_a_git_repo(mock_run, tmp_path): + repo_path = tmp_path / "repo" + repo_path.mkdir() + mock_run.return_value = MagicMock(returncode=128, stdout="") + task = SweBenchVerifiedTask(_instance()) + task.setup(str(repo_path)) + + assert not repo_path.exists() + origin_call, clone_call, checkout_call = mock_run.call_args_list + assert origin_call.args[0] == ["git", "remote", "get-url", "origin"] + assert clone_call.args[0] == [ + "git", "clone", "https://github.com/django/django.git", str(repo_path) + ] @pytest.mark.unit @@ -168,15 +229,6 @@ def test_build_result_includes_dataset_fields(): } -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_teardown_removes_repo_path(mock_run): - task = SweBenchVerifiedTask(_instance()) - task.teardown("/repo") - - mock_run.assert_called_once_with(["rm", "-rf", "/repo"], check=False) - - # --------------------------------------------------------------------------- # SweBenchVerifiedTask.build_feedback # --------------------------------------------------------------------------- @@ -270,10 +322,15 @@ def _fake_run(cmd, **kwargs): if raise_on_harness: raise RuntimeError("harness crashed") run_id = cmd[cmd.index("--run_id") + 1] - report_dir = kwargs["cwd"] + report_dir = Path(kwargs["cwd"]) instance_id = cmd[cmd.index("--instance_ids") + 1] - report = {"resolved_ids": [instance_id] if resolved else []} - (Path(report_dir) / f"microbots-eval-agent.{run_id}.json").write_text(json.dumps(report)) + instance_log_dir = ( + report_dir / "logs" / "run_evaluation" / run_id + / "microbots-eval-agent" / instance_id + ) + instance_log_dir.mkdir(parents=True) + report = {instance_id: {"resolved": resolved}} + (instance_log_dir / "report.json").write_text(json.dumps(report)) return MagicMock(stdout="harness ran\n", stderr="", returncode=0) return MagicMock(stdout="", stderr="", returncode=0) @@ -282,7 +339,7 @@ def _fake_run(cmd, **kwargs): @pytest.mark.unit @patch(f"{MODULE}.subprocess.run") -def test_check_passed_true_when_instance_in_resolved_ids(mock_run, tmp_path): +def test_check_passed_true_when_report_marks_resolved(mock_run, tmp_path): mock_run.side_effect = _make_fake_subprocess_run(resolved=True) log_path = tmp_path / "check.log" log_path.write_text("") @@ -296,7 +353,7 @@ def test_check_passed_true_when_instance_in_resolved_ids(mock_run, tmp_path): @pytest.mark.unit @patch(f"{MODULE}.subprocess.run") -def test_check_passed_false_when_instance_not_in_resolved_ids(mock_run, tmp_path): +def test_check_passed_false_when_report_marks_not_resolved(mock_run, tmp_path): mock_run.side_effect = _make_fake_subprocess_run(resolved=False) log_path = tmp_path / "check.log" log_path.write_text("") @@ -342,6 +399,44 @@ def test_check_appends_to_log_file_without_truncating_existing_content(mock_run, assert "harness ran" in content +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_appends_instance_log_and_test_output_when_present(mock_run, tmp_path): + def _fake_run(cmd, **kwargs): + if cmd[:2] == ["git", "diff"]: + return MagicMock(stdout="diff", stderr="", returncode=0) + if "swebench.harness.run_evaluation" in cmd: + run_id = cmd[cmd.index("--run_id") + 1] + report_dir = Path(kwargs["cwd"]) + instance_id = cmd[cmd.index("--instance_ids") + 1] + + instance_log_dir = ( + report_dir / "logs" / "run_evaluation" / run_id + / "microbots-eval-agent" / instance_id + ) + instance_log_dir.mkdir(parents=True) + report = {instance_id: {"resolved": True}} + (instance_log_dir / "report.json").write_text(json.dumps(report)) + (instance_log_dir / "run_instance.log").write_text("build+test steps") + (instance_log_dir / "test_output.txt").write_text("FAILED test_foo") + + return MagicMock(stdout="harness ran\n", stderr="", returncode=0) + return MagicMock(stdout="", stderr="", returncode=0) + + mock_run.side_effect = _fake_run + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + task.check("/repo", "agent output", str(log_path)) + + content = log_path.read_text() + assert "run_instance.log" in content + assert "build+test steps" in content + assert "test_output.txt" in content + assert "FAILED test_foo" in content + + @pytest.mark.unit @patch(f"{MODULE}.shutil.rmtree") @patch(f"{MODULE}.subprocess.run") @@ -378,7 +473,7 @@ def test_check_cleans_up_even_when_harness_raises(mock_run, mock_rmtree, tmp_pat @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock_memory_tool, tmp_path): +def test_run_calls_setup_build_prompt_check_in_order(mock_bot_cls, mock_memory_tool, tmp_path): from microbots.MicroBot import BotRunResult mock_bot = MagicMock() @@ -393,13 +488,11 @@ def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock calls.append(("check", repo_path, agent_output, log_path)) or CallbackResult(passed=True, reason="ok") ) - task.teardown = lambda repo_path: calls.append(("teardown", repo_path)) outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert calls[0] == ("setup", "/repo") assert calls[1] == ("check", "/repo", "agent did stuff", str(tmp_path / "eval.log")) - assert calls[2] == ("teardown", "/repo") assert outcome.passed is True assert outcome.output == "agent did stuff" @@ -417,7 +510,6 @@ def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_t task = SweBenchVerifiedTask(_instance()) task.setup = lambda repo_path: None task.build_prompt = lambda: "do the task" - task.teardown = lambda repo_path: None seen_log_exists = {} def _check(repo_path, agent_output, log_path): @@ -446,7 +538,6 @@ def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool task.setup = lambda repo_path: None task.build_prompt = lambda: "do the task" task.check = lambda *a: check_calls.append(a) - task.teardown = lambda repo_path: None outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) @@ -461,7 +552,6 @@ def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool def test_run_converts_build_prompt_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool, tmp_path): task = SweBenchVerifiedTask(_instance()) task.setup = lambda repo_path: None - task.teardown = lambda repo_path: None def _build_prompt(): raise ValueError("bad prompt") @@ -489,7 +579,6 @@ def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memor task = SweBenchVerifiedTask(_instance()) task.setup = lambda repo_path: None task.build_prompt = lambda: "do the task" - task.teardown = lambda repo_path: None def _check(repo_path, agent_output, log_path): raise RuntimeError("check exploded") @@ -502,51 +591,6 @@ def _check(repo_path, agent_output, log_path): assert "check exploded" in outcome.result.reason -@pytest.mark.unit -@patch(f"{MODULE}.MemoryTool") -@patch(f"{MODULE}.WritingBot") -def test_run_still_calls_teardown_when_body_raises(mock_bot_cls, mock_memory_tool, tmp_path): - mock_bot_cls.side_effect = RuntimeError("bot construction failed") - - task = SweBenchVerifiedTask(_instance()) - teardown_calls = [] - task.setup = lambda repo_path: None - task.build_prompt = lambda: "do the task" - task.teardown = lambda repo_path: teardown_calls.append(repo_path) - - task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) - - assert teardown_calls == ["/repo"] - - -@pytest.mark.unit -@patch(f"{MODULE}.MemoryTool") -@patch(f"{MODULE}.WritingBot") -def test_run_teardown_exception_does_not_clobber_returned_outcome(mock_bot_cls, mock_memory_tool, tmp_path): - from microbots.MicroBot import BotRunResult - - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) - mock_bot_cls.return_value = mock_bot - - task = SweBenchVerifiedTask(_instance()) - task.setup = lambda repo_path: None - task.build_prompt = lambda: "do the task" - task.check = lambda repo_path, agent_output, log_path: CallbackResult(passed=True, reason="ok") - - def _teardown(repo_path): - raise RuntimeError("teardown boom") - - task.teardown = _teardown - - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) - - # teardown() raised, but the already-computed EvalOutcome must still be returned - assert outcome.passed is True - - - - # --------------------------------------------------------------------------- # SweBenchVerifiedTask.from_config # --------------------------------------------------------------------------- diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py index fba3829..2cabafb 100644 --- a/test/auto_memory/test_orchestrator.py +++ b/test/auto_memory/test_orchestrator.py @@ -13,7 +13,6 @@ from microbots.auto_memory.orchestrator import ( LoopResult, clone_repo, - reset_repo, run, run_train_eval_loop, run_training_loop, @@ -55,7 +54,7 @@ def test_loop_returns_immediately_when_first_round_passes(mock_run_training_loop task = _make_task() task.run.return_value = _make_outcome(passed=True) - result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert isinstance(result, LoopResult) assert result.passed is True @@ -75,7 +74,7 @@ def test_loop_retrains_and_continues_on_failure_then_passes(mock_run_training_lo ] task.build_feedback.return_value = "feedback text" - result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert result.passed is True assert result.rounds_run == 2 @@ -99,7 +98,7 @@ def test_loop_exhausts_max_rounds_without_passing(mock_run_training_loop, tmp_pa ] task.build_feedback.return_value = "feedback text" - result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=3) + result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=3) assert result.passed is False assert result.rounds_run == 3 @@ -116,7 +115,7 @@ def test_log_path_persists_after_passing_round(mock_run_training_loop, tmp_path) task = _make_task() task.run.return_value = _make_outcome(passed=True) - run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert Path(log_path).exists() @@ -133,7 +132,7 @@ def test_log_path_persists_after_failing_round(mock_run_training_loop, tmp_path) ] task.build_feedback.return_value = "feedback text" - run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert Path(log1).exists() assert Path(log2).exists() @@ -150,7 +149,7 @@ def test_build_feedback_exception_does_not_crash_loop(mock_run_training_loop, tm ] task.build_feedback.side_effect = RuntimeError("analysis bot crashed") - result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert result.passed is True assert result.rounds_run == 2 @@ -169,7 +168,7 @@ def test_loop_forwards_training_iterations_to_run_training_loop(mock_run_trainin task.build_feedback.return_value = "feedback text" run_train_eval_loop( - "/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5, training_iterations=4 + "/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5, training_iterations=4 ) mock_run_training_loop.assert_called_once_with( @@ -193,7 +192,7 @@ def test_run_training_exception_does_not_crash_loop(mock_run_training_loop, tmp_ task.build_feedback.return_value = "feedback text" mock_run_training_loop.side_effect = RuntimeError("training crashed") - result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert result.passed is True assert result.rounds_run == 2 @@ -258,19 +257,6 @@ def test_clone_repo_is_noop_when_already_present(mock_run, tmp_path): mock_run.assert_not_called() -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_reset_repo_runs_hard_reset_then_clean(mock_run, tmp_path): - repo_path = tmp_path / "repo" - - reset_repo(repo_path, "abc123") - - assert mock_run.call_args_list == [ - call(["git", "reset", "--hard", "abc123"], cwd=repo_path, check=True), - call(["git", "clean", "-fd"], cwd=repo_path, check=True), - ] - - @pytest.mark.unit def test_write_eval_result_writes_task_build_result_as_json(tmp_path): task = MagicMock() @@ -307,7 +293,7 @@ def test_loop_writes_eval_result_for_every_round(tmp_path): task.build_feedback.return_value = "feedback text" with patch(f"{MODULE}.run_training_loop"): - run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert eval_result_path(tmp_path, 1, "task-1").exists() assert eval_result_path(tmp_path, 2, "task-1").exists() @@ -347,7 +333,8 @@ def test_run_calls_run_train_eval_loop_when_task_given(mock_run_train_eval_loop, ) mock_run_train_eval_loop.assert_called_once_with( - repo_path=str(tmp_path / "repo"), + training_repo_path=str(tmp_path / "repo"), + eval_repo_path=str(tmp_path / "eval_repo"), workdir=tmp_path, model="azure-openai/gpt-4o", task=fake_task, @@ -408,6 +395,24 @@ def fake_train(repo_path, feedback, memory_dir, model, iterations=1): assert (memory_dir(tmp_path) / "notes.md").read_text() == "learned something" +@pytest.mark.unit +@patch(f"{MODULE}.run_training_loop") +def test_run_preserves_original_memory_as_a_seed_snapshot(mock_run_training_loop, tmp_path): + memory_dir(tmp_path).mkdir(parents=True) + (memory_dir(tmp_path) / "notes.md").write_text("original seed") + + def fake_train(repo_path, feedback, memory_dir, model, iterations=1): + Path(memory_dir, "notes.md").write_text("overwritten by training") + + mock_run_training_loop.side_effect = fake_train + + run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) + + assert (memory_dir(tmp_path) / "notes.md").read_text() == "overwritten by training" + assert (tmp_path / "memory_seed" / "notes.md").read_text() == "original seed" + + + @pytest.mark.unit @patch(f"{MODULE}.run_training_loop") def test_loop_carries_memory_forward_between_rounds(mock_run_training_loop, tmp_path): @@ -426,6 +431,6 @@ def fake_run(repo_path, memory_dir, model, log_path): task.run.side_effect = fake_run task.build_feedback.return_value = "feedback text" - run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert (memory_dir(tmp_path) / "notes.md").read_text() == "round 2 progress" diff --git a/test/auto_memory/test_task.py b/test/auto_memory/test_task.py index b66deb1..e04d0e9 100644 --- a/test/auto_memory/test_task.py +++ b/test/auto_memory/test_task.py @@ -13,6 +13,10 @@ class _RunOnlyTask(EvalTask): """A task that overrides only run()/build_feedback(), never touching the optional hooks.""" + @classmethod + def from_config(cls, task_args): + return [cls()] + def run(self, repo_path, memory_dir, model, log_path): return EvalOutcome( passed=True, @@ -33,6 +37,10 @@ def test_run_is_abstract(): @pytest.mark.unit def test_build_feedback_is_abstract(): class _MissingBuildFeedback(EvalTask): + @classmethod + def from_config(cls, task_args): + return [cls()] + def run(self, repo_path, memory_dir, model, log_path): raise NotImplementedError @@ -40,6 +48,25 @@ def run(self, repo_path, memory_dir, model, log_path): _MissingBuildFeedback() +@pytest.mark.unit +def test_from_config_is_abstract(): + class _MissingFromConfig(EvalTask): + def run(self, repo_path, memory_dir, model, log_path): + raise NotImplementedError + + def build_feedback(self, outcome, repo_path, model, log_path): + raise NotImplementedError + + with pytest.raises(TypeError): + _MissingFromConfig() + + +@pytest.mark.unit +def test_from_config_default_body_raises_not_implemented_error(): + with pytest.raises(NotImplementedError): + EvalTask.from_config({}) + + @pytest.mark.unit def test_subclass_overriding_only_run_is_instantiable(): task = _RunOnlyTask() diff --git a/test/auto_memory/test_task_registry.py b/test/auto_memory/test_task_registry.py index ea34cf8..8de0d48 100644 --- a/test/auto_memory/test_task_registry.py +++ b/test/auto_memory/test_task_registry.py @@ -19,6 +19,10 @@ class _DummyTask(EvalTask): def __init__(self, value=None): self.value = value + @classmethod + def from_config(cls, task_args): + return [cls(**task_args)] + def setup(self, repo_path): pass diff --git a/test/auto_memory/test_workdir.py b/test/auto_memory/test_workdir.py index 1eb2fae..96a04af 100644 --- a/test/auto_memory/test_workdir.py +++ b/test/auto_memory/test_workdir.py @@ -12,6 +12,7 @@ eval_dir, eval_log_path, eval_patch_path, + eval_repo_dir, eval_result_path, load_config, load_round_memory, @@ -22,8 +23,8 @@ round_dir, round_log_path, round_memory_dir, - run_log_path, save_round_memory, + snapshot_seed_memory, ) @@ -104,6 +105,41 @@ def test_save_round_memory_overwrites_stale_top_level_files(tmp_path): assert (top_memory / "notes.md").read_text() == "new" +@pytest.mark.unit +def test_snapshot_seed_memory_creates_empty_dir_when_no_top_level_memory(tmp_path): + result = snapshot_seed_memory(tmp_path) + + assert result == tmp_path / "memory_seed" + assert result.is_dir() + assert list(result.iterdir()) == [] + + +@pytest.mark.unit +def test_snapshot_seed_memory_copies_current_top_level_memory(tmp_path): + top_memory = memory_dir(tmp_path) + top_memory.mkdir(parents=True) + (top_memory / "notes.md").write_text("original seed") + + result = snapshot_seed_memory(tmp_path) + + assert (result / "notes.md").read_text() == "original seed" + + +@pytest.mark.unit +def test_snapshot_seed_memory_is_a_noop_once_a_snapshot_exists(tmp_path): + top_memory = memory_dir(tmp_path) + top_memory.mkdir(parents=True) + (top_memory / "notes.md").write_text("original seed") + snapshot_seed_memory(tmp_path) + + # Mutate top-level memory as later rounds/instances would. + (top_memory / "notes.md").write_text("overwritten by later training") + + result = snapshot_seed_memory(tmp_path) + + assert (result / "notes.md").read_text() == "original seed" + + @pytest.mark.unit def test_resolve_workdir_defaults_to_cwd(monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) @@ -135,8 +171,8 @@ def test_repo_dir_returns_workdir_repo(tmp_path): @pytest.mark.unit -def test_run_log_path_returns_workdir_run_log(tmp_path): - assert run_log_path(tmp_path) == tmp_path / "run.log" +def test_eval_repo_dir_returns_workdir_eval_repo(tmp_path): + assert eval_repo_dir(tmp_path) == tmp_path / "eval_repo" @pytest.mark.unit From 998040a2f82c95af8c63633227c8d3e212869d92 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Thu, 3 Sep 2026 09:35:34 +0000 Subject: [PATCH 12/21] Resolve copilot comments --- .../auto_memory/eval/swebenchverified.py | 61 ++++++---- src/microbots/auto_memory/evalTask.py | 7 -- src/microbots/auto_memory/orchestrator.py | 56 ++++++++- src/microbots/auto_memory/workdir.py | 23 +++- .../auto_memory/eval/test_swebenchverified.py | 52 +++++++- test/auto_memory/test_orchestrator.py | 115 +++++++++++++++--- test/auto_memory/test_workdir.py | 36 ++++++ 7 files changed, 287 insertions(+), 63 deletions(-) diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index 889693b..613396d 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -16,7 +16,6 @@ from logging import getLogger from pathlib import Path -from datasets import load_dataset from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome, EvalTask from microbots.auto_memory.task_registry import register_task from microbots.bot.LogAnalysisBot import LogAnalysisBot @@ -50,7 +49,20 @@ def _load_dataset_rows(dataset_name: str): ------- datasets.Dataset The loaded ``test`` split. + + Raises + ------ + ImportError + If the optional ``datasets`` package (the ``training`` extra) + isn't installed. """ + try: + from datasets import load_dataset + except ImportError as exc: + raise ImportError( + "SWE-bench-verified evaluation requires the 'training' extra: " + "pip install 'microbots[training]'" + ) from exc return load_dataset(dataset_name, split="test") @@ -226,18 +238,6 @@ def build_result(self, outcome: EvalOutcome) -> dict: def setup(self, repo_path: str) -> None: """Clone the instance's repo, or reset it, to its base commit. - Clones fresh on first use. If ``repo_path`` already exists - (e.g. left behind by a previous round) *and* its ``origin`` - remote matches this instance's repo, it's reset instead of - re-cloned: ``git reset --hard `` followed by - ``git clean -fd`` discards whatever the agent changed, without - the cost of a full re-clone and without deleting the directory - ``build_feedback`` may still need to inspect afterward. If - ``repo_path`` exists but isn't a checkout of this repo (e.g. a - stale directory left over from a different run/task), it's - removed and cloned fresh instead, to avoid silently operating - on the wrong codebase. - Parameters ---------- repo_path : str @@ -283,9 +283,12 @@ def build_prompt(self) -> str: def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackResult: """Verify the agent's patch using the SWE-bench evaluation harness. - Captures the agent's changes as a git diff, submits it as a - prediction to ``swebench.harness.run_evaluation``, and checks - whether the harness marked this instance as resolved. + Captures the agent's changes as a git diff (after marking any + untracked new files intent-to-add, so files the agent newly + created are included in the diff rather than silently + dropped), submits it as a prediction to + ``swebench.harness.run_evaluation``, and checks whether the + harness marked this instance as resolved. Parameters ---------- @@ -306,21 +309,31 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes CallbackResult Whether the harness marked this instance as resolved. """ + subprocess.run( + ["git", "add", "--intent-to-add", "."], cwd=repo_path, check=True + ) diff = subprocess.run( - ["git", "diff"], cwd=repo_path, capture_output=True, text=True + ["git", "diff", "--binary"], + cwd=repo_path, + capture_output=True, + text=True, + check=True, ).stdout run_id = f"microbots-{uuid.uuid4().hex[:8]}" model_name_or_path = EVAL_AGENT_MODEL_NAME - pred_path = Path(tempfile.mktemp(suffix=".json")) #will need to update when upgraded to ~5.0.2 , removed this flag in the new version #https://github.com/SWE-bench/SWE-bench/commit/e2c13307b6cf7764a50958b9c8bfbfb3f72cb70a report_dir = Path(tempfile.mkdtemp()) - pred_path.write_text(json.dumps([{ - "instance_id": self.instance.instance_id, - "model_patch": diff, - "model_name_or_path": model_name_or_path, - }])) + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) as pred_file: + pred_path = Path(pred_file.name) + pred_file.write(json.dumps([{ + "instance_id": self.instance.instance_id, + "model_patch": diff, + "model_name_or_path": model_name_or_path, + }])) try: proc = subprocess.run( @@ -425,11 +438,11 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva The result of this eval round, including the agent's output, the check verdict. """ - self.setup(repo_path) Path(log_path).parent.mkdir(parents=True, exist_ok=True) Path(log_path).write_text("") try: + self.setup(repo_path) prompt = self.build_prompt() bot = WritingBot( model=model, diff --git a/src/microbots/auto_memory/evalTask.py b/src/microbots/auto_memory/evalTask.py index aacee20..be3af99 100644 --- a/src/microbots/auto_memory/evalTask.py +++ b/src/microbots/auto_memory/evalTask.py @@ -154,13 +154,6 @@ def teardown(self, repo_path: str) -> None: def from_config(cls, task_args: dict[str, Any]) -> list["EvalTask"]: """Required. Build task instance(s) from a config's ``task_args`` dict. - Called by the CLI at runtime (driven by ``--task``) to - construct the actual task object(s) to run, using whatever - config values the task needs (e.g. a dataset instance ID, a - repo filter). Object creation must go through this method - rather than being constructed elsewhere, so behavior stays - driven by the CLI/config at runtime. - Parameters ---------- task_args : dict[str, Any] diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index 7dc5662..e13ad3e 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -9,6 +9,7 @@ from logging import getLogger from pathlib import Path import json +import shutil import subprocess from microbots.auto_memory.evalTask import EvalOutcome, EvalTask @@ -48,18 +49,39 @@ class LoopResult: outcomes: list[EvalOutcome] = field(default_factory=list) def clone_repo(url: str, repo_path: Path) -> None: - """Clone ``url`` into ``repo_path`` if it isn't already cloned there. + """Clone ``url`` into ``repo_path``, or reuse it if already cloned from ``url``. + + Existence alone isn't enough to trust ``repo_path``: it could be an + empty/partial directory left by a previous failed clone, or a + reused workdir whose config now points at a different ``url``. So + if ``repo_path`` exists, its ``origin`` remote is checked against + ``url`` first. Only an exact match is reused as-is; anything else + (mismatched origin, or not a git checkout at all) is removed and + re-cloned, so training never silently runs against missing or + wrong code. Parameters ---------- url : str Git URL (or local path) to clone from. repo_path : Path - Destination directory for the clone. If it already exists (e.g. - a previous round already cloned here), this is a no-op. + Destination directory for the clone. """ if repo_path.exists(): - return + origin = subprocess.run( + ["git", "remote", "get-url", "origin"], + cwd=repo_path, capture_output=True, text=True, + ) + if origin.returncode == 0 and origin.stdout.strip() == url: + return + + logger.warning( + "clone_repo: %s exists but isn't a checkout of %s (origin=%r); " + "removing and re-cloning", + repo_path, url, origin.stdout.strip(), + ) + shutil.rmtree(repo_path) + subprocess.run(["git", "clone", url, str(repo_path)], check=True) def write_eval_result(workdir: Path, round_num: int, task: EvalTask, outcome: EvalOutcome) -> None: @@ -180,7 +202,15 @@ def run_train_eval_loop( LoopResult Whether the task passed, how many rounds ran, and every round's outcome. + + Raises + ------ + ValueError + If ``max_rounds`` is less than 1. """ + if max_rounds < 1: + raise ValueError(f"max_rounds must be >= 1, got {max_rounds}") + outcomes: list[EvalOutcome] = [] for round_idx in range(1, max_rounds+1): @@ -275,12 +305,26 @@ def run( ------- LoopResult | None The eval loop's result if ``task`` was given, otherwise ``None``. + + Raises + ------ + ValueError + If ``config`` has no ``repo`` entry. Every run needs a training + checkout (``run_training_loop`` always mounts + ``training_repo_path``, regardless of ``task``), so ``repo`` + must be configured even for tasks like ``SweBenchVerifiedTask`` + that manage their own separate eval checkout. """ if config is None: config = load_config(workdir) repo_url = config.get("repo") - if repo_url: - clone_repo(repo_url, repo_dir(workdir)) + if not repo_url: + raise ValueError( + "config.yaml must specify 'repo' (the training checkout's clone " + "URL); it is required even when the eval task manages its own " + "separate eval repo checkout." + ) + clone_repo(repo_url, repo_dir(workdir)) snapshot_seed_memory(workdir) diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py index 65cb3ff..154699b 100644 --- a/src/microbots/auto_memory/workdir.py +++ b/src/microbots/auto_memory/workdir.py @@ -250,7 +250,11 @@ def load_round_memory(workdir: Path, round_num: int, *, instance_id: str | None """Copy the current top-level memory into this round's own memory dir. Called before a round's training pass, so it starts from whatever - memory the previous round left behind (or empty, on round 1). + memory the previous round left behind (or empty, on round 1). This + round's memory dir is replaced, not merged into: any stale files + left behind by a previous attempt at this same round (e.g. a + crashed/re-run process) are discarded first, so the round always + starts from an exact snapshot of the current top-level memory. Parameters ---------- @@ -269,9 +273,11 @@ def load_round_memory(workdir: Path, round_num: int, *, instance_id: str | None """ src = memory_dir(workdir) dst = round_memory_dir(workdir, round_num, instance_id=instance_id) - dst.mkdir(parents=True, exist_ok=True) + shutil.rmtree(dst, ignore_errors=True) if src.is_dir(): - shutil.copytree(src, dst, dirs_exist_ok=True) + shutil.copytree(src, dst) + else: + dst.mkdir(parents=True, exist_ok=True) return dst @@ -279,7 +285,10 @@ def save_round_memory(workdir: Path, round_num: int, *, instance_id: str | None """Copy this round's memory back up to the top-level memory dir. Called after a round's training pass, so later rounds (and the - final saved memory) see what this round learned. + final saved memory) see what this round learned. The top-level + memory dir is replaced, not merged into: files the round deleted + (e.g. via the agent's ``memory delete`` command) are gone from + the top level too, instead of surviving from a previous save. Parameters ---------- @@ -298,9 +307,11 @@ def save_round_memory(workdir: Path, round_num: int, *, instance_id: str | None """ src = round_memory_dir(workdir, round_num, instance_id=instance_id) dst = memory_dir(workdir) - dst.mkdir(parents=True, exist_ok=True) + shutil.rmtree(dst, ignore_errors=True) if src.is_dir(): - shutil.copytree(src, dst, dirs_exist_ok=True) + shutil.copytree(src, dst) + else: + dst.mkdir(parents=True, exist_ok=True) return dst diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py index 79477ec..7caffea 100644 --- a/test/auto_memory/eval/test_swebenchverified.py +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -11,6 +11,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/"))) from microbots.auto_memory.eval.swebenchverified import ( + SWE_BENCH_VERIFIED, SweBenchInstance, SweBenchVerifiedTask, _load_dataset_rows, @@ -59,7 +60,7 @@ def _fake_rows(): # --------------------------------------------------------------------------- @pytest.mark.unit -@patch(f"{MODULE}.load_dataset") +@patch("datasets.load_dataset") def test_load_instances_of_repo_filters_by_repo(mock_load_dataset): mock_load_dataset.return_value = _fake_rows() @@ -70,7 +71,7 @@ def test_load_instances_of_repo_filters_by_repo(mock_load_dataset): @pytest.mark.unit -@patch(f"{MODULE}.load_dataset") +@patch("datasets.load_dataset") def test_load_instances_of_repo_returns_all_when_repo_none(mock_load_dataset): mock_load_dataset.return_value = _fake_rows() @@ -80,7 +81,7 @@ def test_load_instances_of_repo_returns_all_when_repo_none(mock_load_dataset): @pytest.mark.unit -@patch(f"{MODULE}.load_dataset") +@patch("datasets.load_dataset") def test_load_instance_using_id_returns_matching_instance(mock_load_dataset): mock_load_dataset.return_value = _fake_rows() @@ -91,7 +92,7 @@ def test_load_instance_using_id_returns_matching_instance(mock_load_dataset): @pytest.mark.unit -@patch(f"{MODULE}.load_dataset") +@patch("datasets.load_dataset") def test_load_instance_using_id_raises_when_not_found(mock_load_dataset): mock_load_dataset.return_value = _fake_rows() @@ -100,7 +101,7 @@ def test_load_instance_using_id_raises_when_not_found(mock_load_dataset): @pytest.mark.unit -@patch(f"{MODULE}.load_dataset") +@patch("datasets.load_dataset") def test_dataset_rows_are_cached_across_repeated_calls(mock_load_dataset): """``load_dataset`` should only be called once per ``dataset_name``, even across multiple ``load_instances_of_repo``/``load_instance_using_id`` calls.""" @@ -113,6 +114,13 @@ def test_dataset_rows_are_cached_across_repeated_calls(mock_load_dataset): mock_load_dataset.assert_called_once() +@pytest.mark.unit +def test_load_dataset_rows_raises_helpful_error_when_datasets_not_installed(): + with patch.dict(sys.modules, {"datasets": None}): + with pytest.raises(ImportError, match=r"pip install 'microbots\[training\]'"): + _load_dataset_rows(SWE_BENCH_VERIFIED) + + # --------------------------------------------------------------------------- # SweBenchVerifiedTask.setup / build_prompt # --------------------------------------------------------------------------- @@ -337,6 +345,23 @@ def _fake_run(cmd, **kwargs): return _fake_run +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_marks_untracked_files_intent_to_add_before_diffing(mock_run, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + task.check("/repo", "agent output", str(log_path)) + + calls = [c.args[0] for c in mock_run.call_args_list] + add_idx = calls.index(["git", "add", "--intent-to-add", "."]) + diff_idx = calls.index(["git", "diff", "--binary"]) + assert add_idx < diff_idx + assert mock_run.call_args_list[add_idx].kwargs["cwd"] == "/repo" + + @pytest.mark.unit @patch(f"{MODULE}.subprocess.run") def test_check_passed_true_when_report_marks_resolved(mock_run, tmp_path): @@ -566,6 +591,23 @@ def _build_prompt(): assert "bad prompt" in f.read() +@pytest.mark.unit +def test_run_converts_setup_exception_to_failed_outcome(tmp_path): + task = SweBenchVerifiedTask(_instance()) + + def _setup(repo_path): + raise RuntimeError("clone failed") + + task.setup = _setup + + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) + + assert outcome.passed is False + assert "clone failed" in outcome.result.reason + with open(str(tmp_path / "eval.log")) as f: + assert "clone failed" in f.read() + + @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py index 2cabafb..ba193be 100644 --- a/test/auto_memory/test_orchestrator.py +++ b/test/auto_memory/test_orchestrator.py @@ -48,6 +48,17 @@ def _make_task() -> MagicMock: return task +@pytest.mark.unit +@pytest.mark.parametrize("max_rounds", [0, -1]) +def test_loop_raises_for_non_positive_max_rounds(max_rounds): + task = _make_task() + + with pytest.raises(ValueError, match="max_rounds must be >= 1"): + run_train_eval_loop("/repo", "/eval_repo", Path("/workdir"), "azure-openai/gpt-4o", task, max_rounds=max_rounds) + + task.run.assert_not_called() + + @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") def test_loop_returns_immediately_when_first_round_passes(mock_run_training_loop, tmp_path): @@ -248,13 +259,48 @@ def test_clone_repo_clones_when_missing(mock_run, tmp_path): @pytest.mark.unit @patch(f"{MODULE}.subprocess.run") -def test_clone_repo_is_noop_when_already_present(mock_run, tmp_path): +def test_clone_repo_is_noop_when_origin_matches(mock_run, tmp_path): repo_path = tmp_path / "repo" repo_path.mkdir() + mock_run.return_value = MagicMock(returncode=0, stdout="https://example.com/repo.git\n") clone_repo("https://example.com/repo.git", repo_path) - mock_run.assert_not_called() + mock_run.assert_called_once_with( + ["git", "remote", "get-url", "origin"], + cwd=repo_path, capture_output=True, text=True, + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_clone_repo_removes_and_reclones_when_origin_mismatched(mock_run, tmp_path): + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "stale_marker.txt").write_text("leftover from a different repo") + mock_run.return_value = MagicMock(returncode=0, stdout="https://example.com/other-repo.git\n") + + clone_repo("https://example.com/repo.git", repo_path) + + assert not (repo_path / "stale_marker.txt").exists() + mock_run.assert_called_with( + ["git", "clone", "https://example.com/repo.git", str(repo_path)], check=True + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_clone_repo_removes_and_reclones_when_repo_path_not_a_git_repo(mock_run, tmp_path): + repo_path = tmp_path / "repo" + repo_path.mkdir() + mock_run.return_value = MagicMock(returncode=128, stdout="") + + clone_repo("https://example.com/repo.git", repo_path) + + assert not repo_path.exists() + mock_run.assert_called_with( + ["git", "clone", "https://example.com/repo.git", str(repo_path)], check=True + ) @pytest.mark.unit @@ -304,9 +350,16 @@ def test_loop_writes_eval_result_for_every_round(tmp_path): @pytest.mark.unit +@patch(f"{MODULE}.clone_repo") @patch(f"{MODULE}.run_training_loop") -def test_run_calls_run_training_loop_when_task_is_none(mock_run_training_loop, tmp_path): - result = run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None, training_iterations=2) +def test_run_calls_run_training_loop_when_task_is_none(mock_run_training_loop, mock_clone_repo, tmp_path): + result = run( + workdir=tmp_path, + model="azure-openai/gpt-4o", + task=None, + training_iterations=2, + config={"repo": "https://example.com/repo.git"}, + ) mock_run_training_loop.assert_called_once_with( repo_path=str(tmp_path / "repo"), @@ -319,8 +372,9 @@ def test_run_calls_run_training_loop_when_task_is_none(mock_run_training_loop, t @pytest.mark.unit +@patch(f"{MODULE}.clone_repo") @patch(f"{MODULE}.run_train_eval_loop") -def test_run_calls_run_train_eval_loop_when_task_given(mock_run_train_eval_loop, tmp_path): +def test_run_calls_run_train_eval_loop_when_task_given(mock_run_train_eval_loop, mock_clone_repo, tmp_path): fake_task = MagicMock() mock_run_train_eval_loop.return_value = "loop-result" @@ -330,6 +384,7 @@ def test_run_calls_run_train_eval_loop_when_task_given(mock_run_train_eval_loop, task=fake_task, max_rounds=3, training_iterations=2, + config={"repo": "https://example.com/repo.git"}, ) mock_run_train_eval_loop.assert_called_once_with( @@ -345,19 +400,35 @@ def test_run_calls_run_train_eval_loop_when_task_given(mock_run_train_eval_loop, @pytest.mark.unit +@patch(f"{MODULE}.clone_repo") @patch(f"{MODULE}.run_train_eval_loop") @patch(f"{MODULE}.run_training_loop") -def test_run_does_not_call_eval_loop_when_task_is_none(mock_run_training_loop, mock_run_train_eval_loop, tmp_path): - run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) +def test_run_does_not_call_eval_loop_when_task_is_none( + mock_run_training_loop, mock_run_train_eval_loop, mock_clone_repo, tmp_path +): + run( + workdir=tmp_path, + model="azure-openai/gpt-4o", + task=None, + config={"repo": "https://example.com/repo.git"}, + ) mock_run_train_eval_loop.assert_not_called() @pytest.mark.unit +@patch(f"{MODULE}.clone_repo") @patch(f"{MODULE}.run_train_eval_loop") @patch(f"{MODULE}.run_training_loop") -def test_run_does_not_call_training_loop_when_task_given(mock_run_training_loop, mock_run_train_eval_loop, tmp_path): - run(workdir=tmp_path, model="azure-openai/gpt-4o", task=MagicMock()) +def test_run_does_not_call_training_loop_when_task_given( + mock_run_training_loop, mock_run_train_eval_loop, mock_clone_repo, tmp_path +): + run( + workdir=tmp_path, + model="azure-openai/gpt-4o", + task=MagicMock(), + config={"repo": "https://example.com/repo.git"}, + ) mock_run_training_loop.assert_not_called() @@ -376,28 +447,37 @@ def test_run_clones_repo_from_config_when_repo_url_given(mock_run_training_loop, @pytest.mark.unit @patch(f"{MODULE}.clone_repo") @patch(f"{MODULE}.run_training_loop") -def test_run_does_not_clone_when_config_has_no_repo(mock_run_training_loop, mock_clone_repo, tmp_path): - run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) +def test_run_raises_when_config_has_no_repo(mock_run_training_loop, mock_clone_repo, tmp_path): + with pytest.raises(ValueError, match="repo"): + run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None, config={}) mock_clone_repo.assert_not_called() + mock_run_training_loop.assert_not_called() @pytest.mark.unit +@patch(f"{MODULE}.clone_repo") @patch(f"{MODULE}.run_training_loop") -def test_run_promotes_round1_memory_to_top_level_for_train_only_mode(mock_run_training_loop, tmp_path): +def test_run_promotes_round1_memory_to_top_level_for_train_only_mode(mock_run_training_loop, mock_clone_repo, tmp_path): def fake_train(repo_path, feedback, memory_dir, model, iterations=1): Path(memory_dir, "notes.md").write_text("learned something") mock_run_training_loop.side_effect = fake_train - run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) + run( + workdir=tmp_path, + model="azure-openai/gpt-4o", + task=None, + config={"repo": "https://example.com/repo.git"}, + ) assert (memory_dir(tmp_path) / "notes.md").read_text() == "learned something" @pytest.mark.unit +@patch(f"{MODULE}.clone_repo") @patch(f"{MODULE}.run_training_loop") -def test_run_preserves_original_memory_as_a_seed_snapshot(mock_run_training_loop, tmp_path): +def test_run_preserves_original_memory_as_a_seed_snapshot(mock_run_training_loop, mock_clone_repo, tmp_path): memory_dir(tmp_path).mkdir(parents=True) (memory_dir(tmp_path) / "notes.md").write_text("original seed") @@ -406,7 +486,12 @@ def fake_train(repo_path, feedback, memory_dir, model, iterations=1): mock_run_training_loop.side_effect = fake_train - run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) + run( + workdir=tmp_path, + model="azure-openai/gpt-4o", + task=None, + config={"repo": "https://example.com/repo.git"}, + ) assert (memory_dir(tmp_path) / "notes.md").read_text() == "overwritten by training" assert (tmp_path / "memory_seed" / "notes.md").read_text() == "original seed" diff --git a/test/auto_memory/test_workdir.py b/test/auto_memory/test_workdir.py index 96a04af..4c87baa 100644 --- a/test/auto_memory/test_workdir.py +++ b/test/auto_memory/test_workdir.py @@ -70,6 +70,24 @@ def test_load_round_memory_copies_top_level_memory_into_round(tmp_path): assert (result / "notes.md").read_text() == "prior findings" +@pytest.mark.unit +def test_load_round_memory_discards_stale_files_left_in_round_dir(tmp_path): + # Simulate a previous crashed/re-run attempt at this same round that + # left behind a file no longer present in top-level memory. + round_memory = round_memory_dir(tmp_path, 1) + round_memory.mkdir(parents=True) + (round_memory / "stale.md").write_text("leftover from a crashed attempt") + + top_memory = memory_dir(tmp_path) + top_memory.mkdir(parents=True) + (top_memory / "notes.md").write_text("current memory") + + result = load_round_memory(tmp_path, 1) + + assert (result / "notes.md").read_text() == "current memory" + assert not (result / "stale.md").exists() + + @pytest.mark.unit def test_save_round_memory_creates_empty_dir_when_no_round_memory(tmp_path): result = save_round_memory(tmp_path, 1) @@ -105,6 +123,24 @@ def test_save_round_memory_overwrites_stale_top_level_files(tmp_path): assert (top_memory / "notes.md").read_text() == "new" +@pytest.mark.unit +def test_save_round_memory_propagates_deletions_to_top_level(tmp_path): + # The agent deleted a file during this round (e.g. via `memory + # delete`); the top level shouldn't resurrect it from a prior save. + top_memory = memory_dir(tmp_path) + top_memory.mkdir(parents=True) + (top_memory / "stale.md").write_text("no longer relevant") + + round_memory = round_memory_dir(tmp_path, 1) + round_memory.mkdir(parents=True) + (round_memory / "notes.md").write_text("kept") + + save_round_memory(tmp_path, 1) + + assert not (top_memory / "stale.md").exists() + assert (top_memory / "notes.md").read_text() == "kept" + + @pytest.mark.unit def test_snapshot_seed_memory_creates_empty_dir_when_no_top_level_memory(tmp_path): result = snapshot_seed_memory(tmp_path) From db258daa0d9e659a769b257f97d9b671c688cfb7 Mon Sep 17 00:00:00 2001 From: bala Date: Mon, 7 Sep 2026 15:18:30 +0000 Subject: [PATCH 13/21] Single training should be validated by all the instances in the task --- src/microbots/auto_memory/__init__.py | 2 +- src/microbots/auto_memory/architecture.md | 196 ++++++++++ src/microbots/auto_memory/cli.py | 42 ++- .../auto_memory/eval/swebenchverified.py | 357 +++++++++++------- src/microbots/auto_memory/evalTask.py | 159 ++------ src/microbots/auto_memory/orchestrator.py | 84 ++--- src/microbots/auto_memory/task_registry.py | 32 +- src/microbots/auto_memory/workdir.py | 38 +- src/microbots/auto_memory/workdir/config.yaml | 3 + 9 files changed, 523 insertions(+), 390 deletions(-) create mode 100644 src/microbots/auto_memory/architecture.md create mode 100644 src/microbots/auto_memory/workdir/config.yaml diff --git a/src/microbots/auto_memory/__init__.py b/src/microbots/auto_memory/__init__.py index e959bfa..e967e45 100644 --- a/src/microbots/auto_memory/__init__.py +++ b/src/microbots/auto_memory/__init__.py @@ -4,5 +4,5 @@ an evaluation task and run it in a loop against a training agent. """ -from .evalTask import CallbackResult, EvalOutcome, EvalTask +from .evalTask import EvalOutcome, EvalTask from .orchestrator import LoopResult, run_train_eval_loop \ No newline at end of file diff --git a/src/microbots/auto_memory/architecture.md b/src/microbots/auto_memory/architecture.md new file mode 100644 index 0000000..01c9135 --- /dev/null +++ b/src/microbots/auto_memory/architecture.md @@ -0,0 +1,196 @@ +# auto_memory — Architecture + +An agent that **learns a repository into memory notes**, then **proves those notes work** by +solving a real task with them. If it fails, it learns again from the failure and retries. + +> Train → Eval → Feedback → Train → … until pass (or rounds run out). + +--- + +## 1. The Big Picture + +```mermaid +flowchart LR + subgraph LOOP["Train / Eval Loop"] + direction TB + T["🧠 TRAIN
ReadingBot reads the repo
writes notes to memory/"] + E["🎯 EVAL
WritingBot solves a task
using only those notes"] + C{"Passed?"} + F["🔍 FEEDBACK
LogAnalysisBot reads the failure log
says what the notes were missing"] + + E --> C + C -- "yes" --> DONE(["✅ Done"]) + C -- "no" --> F --> T --> E + end + + CLI["cli.py
--model --task --max-rounds"] --> LOOP +``` + +**Key idea:** the eval agent gets *no* extra hints — only the memory notes. +So a failing eval is direct evidence the notes are wrong or incomplete. + +--- + +## 2. The Cast + +| File | Role | One-liner | +|---|---|---| +| `cli.py` | Entry point | Parses args, builds tasks, calls the orchestrator | +| `orchestrator.py` | Conductor | Owns the round loop, clones repo, wires train ↔ eval | +| `evalTask.py` | Contract | Abstract `EvalTask`: `run`, `check`, `build_feedback`, … | +| `task_registry.py` | Plugin table | `@register_task("name")` + auto-import of `eval/*` | +| `eval/swebenchverified.py` | A real task | One SWE-bench-Verified issue, graded by the official harness | +| `training/runner.py` | Trainer | One `ReadingBot` pass + `MemoryTool` | +| `training/training_instructions.md` | Trainer's brief | "Learn the repo, write notes, never edit code" | +| `workdir.py` | Filing clerk | Every path under `workdir/` lives here — nothing is hard-coded elsewhere | + +--- + +## 3. One Round, Step by Step + +```mermaid +sequenceDiagram + autonumber + participant O as orchestrator + participant W as workdir + participant Task as EvalTask + participant Bot as WritingBot + participant Train as run_training_loop + + O->>W: load_round_memory(round N) + Note over W: copy memory/ ➜ rounds_/round_N/memory + O->>Task: run(eval_repo, memory_dir, model, log) + Task->>Task: setup() – clone/reset repo @ base commit + Task->>Bot: build_prompt() + MemoryTool(memory_dir) + Bot-->>Task: patch in repo + output + Task->>Task: check() – grade it (SWE-bench harness) + Task-->>O: EvalOutcome(passed, output, result) + + alt passed + O-->>O: return LoopResult(passed=True) + else failed + O->>Task: build_feedback(outcome, log) + Task-->>O: "your notes were missing X" + O->>Train: run_training_loop(feedback, memory_dir) × iterations + Train-->>W: notes updated in place + end + + O->>W: write result.json + save_round_memory(round N) + Note over W: copy round memory ➜ back up to memory/ +``` + +--- + +## 4. Memory Lifecycle (the heart of it) + +Memory is a **directory of markdown notes** that is copied down into each round, +mutated by the bots, then copied back up. + +```mermaid +flowchart TD + SEED["workdir/memory_seed/
immutable baseline snapshot"] + TOP["workdir/memory/
current best notes"] + R1["round_1/memory"] + R2["round_2/memory"] + R3["round_N/memory"] + + TOP -. "snapshot once, at run start" .-> SEED + TOP -->|load_round_memory| R1 + R1 -->|save_round_memory| TOP + TOP -->|load_round_memory| R2 + R2 -->|save_round_memory| TOP + TOP -->|load_round_memory| R3 + R3 -->|save_round_memory| TOP +``` + +Rules that matter: + +- **Replace, never merge.** `load`/`save` do `rmtree` + `copytree`, so deleted notes stay deleted + and stale files from a crashed round can't leak in. +- **`memory_seed` is written once.** It preserves the pre-run state, because `memory/` is + mutated in place all run long. +- **Memory carries across tasks.** Multiple task instances in one workdir share `memory/`, + so later instances inherit what earlier ones learned. + +--- + +## 5. Workdir Layout + +```text +workdir/ +├── config.yaml # repo URL + task_args +├── repo/ # persistent clone — TRAINING only +├── eval_repo/ # task-managed clone — EVAL only (reset each round) +├── memory_seed/ # baseline snapshot (write-once) +├── memory/ # current best notes ← the thing being optimized +└── rounds_/ # per-task-instance, so instances never collide + └── round_N/ + ├── memory/ # this round's working copy of the notes + └── eval/ + ├── eval.log # agent output + harness logs (feeds LogAnalysisBot) + └── result.json +``` + +Two repos on purpose: the eval task wipes/resets its checkout every round, which +would otherwise destroy the training checkout. + +--- + +## 6. Two Modes + +```mermaid +flowchart LR + A["orchestrator.run(task=?)"] + A -->|"task is None"| B["Train-only
N training passes, empty feedback
round 1 is just a scratch dir"] + A -->|"task given"| C["run_train_eval_loop
up to max_rounds"] +``` + +```bash +# train only +python -m microbots.auto_memory.cli --model azure-openai/gpt-5.5 + +# train + eval against a SWE-bench instance +python -m microbots.auto_memory.cli \ + --model azure-openai/gpt-5.5 \ + --task swebenchverified \ + --max-rounds 5 --training-iterations 10 +``` + +--- + +## 7. Adding a New Eval Task + +Drop a module in `eval/` — `discover_tasks()` imports everything in that package, +so the `@register_task` decorator fires and the name shows up in `--task`. No central +factory to edit. + +```python +@register_task("mytask") +class MyTask(EvalTask): + @classmethod + def from_config(cls, task_args: dict) -> list["EvalTask"]: + ... # one instance per unit of work + + def run(self, repo_path, memory_dir, model, log_path) -> EvalOutcome: + ... # you drive setup/build_prompt/check yourself + + def build_feedback(self, outcome, repo_path, model, log_path) -> str: + ... # turn the failure log into "what the notes should say" +``` + +Required: `from_config`, `run`, `build_feedback`. +Optional hooks (`setup`, `build_prompt`, `check`, `teardown`, `build_result`, `task_id`) +are **not** called automatically — your `run` decides. + +--- + +## 8. Failure Handling at a Glance + +| Where it breaks | What happens | +|---|---| +| Agent run raises | Caught in `run`; logged; round fails with the exception as the reason | +| `build_feedback` / retraining raises | Logged; loop **continues to the next round** without retraining | +| Repo dir exists with wrong `origin` | Removed and re-cloned (never silently trains on wrong code) | +| `max_rounds` exhausted | `LoopResult(passed=False)` with every round's outcome | + +Whatever happens, the `finally` block still writes `result.json` and saves the round's memory. diff --git a/src/microbots/auto_memory/cli.py b/src/microbots/auto_memory/cli.py index 66ab0e6..383f313 100644 --- a/src/microbots/auto_memory/cli.py +++ b/src/microbots/auto_memory/cli.py @@ -4,7 +4,6 @@ - ``--task `` given: run the full train <-> eval loop for that task. -- ``--task`` omitted: train only, no eval task, with empty feedback. Both modes are dispatched via ``orchestrator.run``. """ @@ -15,7 +14,7 @@ from microbots.auto_memory.orchestrator import run from microbots.auto_memory.task_registry import TASK_REGISTRY, discover_tasks -from microbots.auto_memory.workdir import load_config, require_workdir, resolve_workdir +from microbots.auto_memory.workdir import require_workdir, resolve_workdir logger = logging.getLogger(__name__) @@ -47,8 +46,12 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) parser.add_argument( "--task", + required=True, choices=sorted(TASK_REGISTRY), - help="Eval task to run. Omit to only run training, with no eval task.", + help="Eval task to run.", + ) + parser.add_argument( + "--config-file", type=Path, help="Path to the task configuration file.", ) parser.add_argument("--max-rounds", type=int, default=5) parser.add_argument("--training-iterations", type=int, default=10) @@ -68,25 +71,24 @@ def main(argv: list[str] | None = None) -> None: workdir = Path(args.workdir) if args.workdir else resolve_workdir() require_workdir(workdir) - config = load_config(workdir) - tasks = ( - TASK_REGISTRY[args.task].from_config(config.get("task_args", {})) - if args.task - else [None] + if not args.config_file: + config_file = workdir / "task_config.yaml" + else: + config_file = args.config_file + if not config_file.is_file(): + raise FileNotFoundError(f"Config file not found: {config_file}") + + result = run( + workdir=workdir, + model=args.model, + task=TASK_REGISTRY[args.task](config_file=config_file), + max_rounds=args.max_rounds, + training_iterations=args.training_iterations, ) - for task in tasks: - result = run( - workdir=workdir, - model=args.model, - task=task, - max_rounds=args.max_rounds, - training_iterations=args.training_iterations, - config=config, + if result is not None: + logger.info( + "task=%s passed=%s rounds_run=%d", args.task, result.passed, result.rounds_run ) - if result is not None: - logger.info( - "task=%s passed=%s rounds_run=%d", args.task, result.passed, result.rounds_run - ) if __name__ == "__main__": main() diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index 613396d..31d2866 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -12,13 +12,15 @@ import tempfile import uuid from dataclasses import dataclass -from functools import lru_cache +from functools import cache from logging import getLogger from pathlib import Path -from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome, EvalTask +import yaml + +from microbots.auto_memory.evalTask import EvalOutcome, EvalTask from microbots.auto_memory.task_registry import register_task -from microbots.bot.LogAnalysisBot import LogAnalysisBot +from microbots.bot.ReadingBot import ReadingBot from microbots.bot.WritingBot import WritingBot from microbots.MicroBot import BotRunResult from microbots.tools.tool_definitions.memory_tool import MemoryTool @@ -29,14 +31,36 @@ EVAL_AGENT_MODEL_NAME = "microbots-eval-agent" -@lru_cache(maxsize=None) +@dataclass +class SweBenchInstance: + """A single SWE-bench-verified dataset row. + + Attributes + ---------- + instance_id : str + Unique identifier for the instance, e.g. ``"django__django-11099"``. + repo : str + The GitHub repo this instance belongs to, e.g. ``"django/django"``. + base_commit : str + Commit hash representing the repo state before the issue's fix. + problem_statement : str + The GitHub issue title and body describing the bug to fix. + """ + + instance_id: str + repo: str + base_commit: str + problem_statement: str + + +@cache def _load_dataset_rows(dataset_name: str): """Load and cache ``dataset_name``'s ``test`` split for the process's lifetime. ``load_dataset`` caches the downloaded files on disk, but still re-reads and rebuilds the in-memory ``Dataset`` object on every call. Since ``load_instances_of_repo``/``load_instance_using_id`` - may each be called many times (e.g. once per eval task instance), + may each be called many times, this wraps ``load_dataset`` with an in-memory cache keyed by ``dataset_name``, so the dataset is only loaded once per process. @@ -69,7 +93,7 @@ def _load_dataset_rows(dataset_name: str): def load_instances_of_repo( dataset_name: str = SWE_BENCH_VERIFIED, repo: str | None = None, -) -> list["SweBenchInstance"]: +) -> list[SweBenchInstance]: """Load all dataset instances, optionally filtered to a single repo. Parameters @@ -100,7 +124,7 @@ def load_instances_of_repo( ] return instances -def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIFIED) -> "SweBenchInstance": +def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIFIED) -> SweBenchInstance: """Load a single dataset instance by its instance ID. Parameters @@ -133,30 +157,8 @@ def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIF ) raise ValueError(f"instance_id not found: {instance_id}") -@dataclass -class SweBenchInstance: - """A single SWE-bench-verified dataset row. - - Attributes - ---------- - instance_id : str - Unique identifier for the instance, e.g. ``"django__django-11099"``. - repo : str - The GitHub repo this instance belongs to, e.g. ``"django/django"``. - base_commit : str - Commit hash representing the repo state before the issue's fix. - problem_statement : str - The GitHub issue title and body describing the bug to fix. - """ - - instance_id: str - repo: str - base_commit: str - problem_statement: str - -@register_task("swebenchverified") -class SweBenchVerifiedTask(EvalTask): - """Eval task that verifies a fix against one SWE-bench-verified instance. +class SweBenchVerifiedTask_one(): + """SWE-bench-verified based evaluation task. Checks out the instance's repo at its base commit, gives the agent the issue's problem statement, and verifies the agent's patch using @@ -180,28 +182,6 @@ def __init__(self, instance: SweBenchInstance | None = None): """ self.instance = instance - @classmethod - def from_config(cls, task_args: dict) -> list["SweBenchVerifiedTask"]: - """Build task(s) from a config's ``task_args`` dict. - - Parameters - ---------- - task_args : dict - Task-specific config values, expected to include - ``instance_id`` and/or ``swebench_repo``. - - Returns - ------- - list[SweBenchVerifiedTask] - One task per matching dataset instance. A single-element - list when ``instance_id`` is given. - """ - if task_args.get("instance_id"): - instances = [load_instance_using_id(task_args["instance_id"])] - else: - instances = load_instances_of_repo(repo=task_args.get("swebench_repo")) - return [cls(instance) for instance in instances] - @property def task_id(self) -> str: """Return this instance's SWE-bench-verified ``instance_id``. @@ -213,28 +193,6 @@ def task_id(self) -> str: """ return self.instance.instance_id - def build_result(self, outcome: EvalOutcome) -> dict: - """Summarize a round's outcome, including the instance's dataset fields. - - Parameters - ---------- - outcome : EvalOutcome - The round's outcome to summarize. - - Returns - ------- - dict - ``passed``/``reason`` plus ``instance_id``, ``repo``, and - ``base_commit`` identifying which dataset row this is. - """ - return { - "passed": outcome.result.passed, - "reason": outcome.result.reason, - "instance_id": self.instance.instance_id, - "repo": self.instance.repo, - "base_commit": self.instance.base_commit, - } - def setup(self, repo_path: str) -> None: """Clone the instance's repo, or reset it, to its base commit. @@ -280,7 +238,7 @@ def build_prompt(self) -> str: """ return self.instance.problem_statement - def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackResult: + def check(self, repo_path: str, agent_output: str, log_path: str) -> BotRunResult: """Verify the agent's patch using the SWE-bench evaluation harness. Captures the agent's changes as a git diff (after marking any @@ -306,8 +264,12 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes Returns ------- - CallbackResult - Whether the harness marked this instance as resolved. + BotRunResult + ``status`` is whether the harness marked this instance as + resolved. On failure, ``error`` carries the harness's + ``test_output.txt`` (or its console output, if the harness + died before producing one) so the feedback bot can see why + the tests failed. """ subprocess.run( ["git", "add", "--intent-to-add", "."], cwd=repo_path, check=True @@ -353,12 +315,17 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes report_dir / "logs" / "run_evaluation" / run_id / model_name_or_path / self.instance.instance_id ) + # Read while report_dir still exists; the finally block deletes it. + test_output = "" with open(log_path, "a") as f: f.write(proc.stdout + proc.stderr) for log_filename in ("run_instance.log", "test_output.txt"): log_file = instance_log_dir / log_filename if log_file.exists(): - f.write(f"\n--- {log_filename} ---\n{log_file.read_text()}\n") + content = log_file.read_text() + if log_filename == "test_output.txt": + test_output = content + f.write(f"\n--- {log_filename} ---\n{content}\n") report_file = instance_log_dir / "report.json" passed = False @@ -369,54 +336,14 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes pred_path.unlink(missing_ok=True) shutil.rmtree(report_dir, ignore_errors=True) - return CallbackResult(passed=passed, reason="resolved" if passed else "not resolved") - - def build_feedback(self, outcome: EvalOutcome, repo_path: str, model: str, log_path: str) -> str: - """Analyze a failed round's log via ``LogAnalysisBot`` for training feedback. - - Parameters - ---------- - outcome : EvalOutcome - The failed outcome to analyze. - repo_path : str - Absolute path to the repo the task was evaluated against. - model : str - The model to use, in the format ``/``. - log_path : str - Path to the round's log file (the same path passed to - ``run``), analyzed by ``LogAnalysisBot``. - - Returns - ------- - str - Feedback text describing the root cause of the failure and - what the agent's memory notes should cover next time. - """ - bot = LogAnalysisBot(model=model, folder_to_mount=repo_path) - result: BotRunResult = bot.run( - file_name=log_path, - user_prompt=( - "This log was produced while verifying whether an " - "agent completed its task correctly. Identify " - "the root cause of the failure and describe concretely " - "what the agent's memory notes should cover next time to " - "avoid this failure." - ), + return BotRunResult( + status = passed, + result = "resolved" if passed else "not resolved", + # Harness can fail before producing test_output.txt; fall back to its console output. + error = None if passed else (test_output or proc.stdout + proc.stderr) ) - if result.status and result.result: - return result.result - - logger.warning( - "LogAnalysisBot failed to analyze failure (%s); falling back to plain feedback", - result.error, - ) - return ( - f"Evaluation failed. Agent output: {outcome.output}\n" - f"Callback reason: {outcome.result.reason}" - ) - - def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> EvalOutcome: + def eval(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> BotRunResult: """Run one eval iteration: setup -> build_prompt -> WritingBot -> check. Parameters @@ -434,7 +361,7 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva Returns ------- - EvalOutcome + BotRunResult The result of this eval round, including the agent's output, the check verdict. """ @@ -454,31 +381,171 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva with open(log_path, "a") as f: f.write(f"Agent output:\n{bot_result.result}\n") - if not bot_result.status: - reason = f"Bot run failed: {bot_result.error}" - with open(log_path, "a") as f: - f.write(f"\n{reason}\n") - result = CallbackResult(passed=False, reason=reason) - else: - result = self.check(repo_path, bot_result.result or "", log_path) + return bot_result - return EvalOutcome( - passed=result.passed, - output=bot_result.result, - result=result, - ) except Exception as exc: logger.exception( - "SweBenchVerifiedTask.run: iteration raised %s", type(exc).__name__ + "SweBenchVerifiedTask.eval: iteration raised %s", type(exc).__name__ ) with open(log_path, "a") as f: f.write(f"\nException during eval iteration: {type(exc).__name__}: {exc}\n") - return EvalOutcome( - passed=False, - output=None, - result=CallbackResult( - passed=False, reason=f"{type(exc).__name__}: {exc}" - ), + return BotRunResult( + status=False, + result=None, + error=f"{type(exc).__name__}: {exc}" ) +@register_task("swebenchverified") +class SweBenchVerified(EvalTask): + """SWE-bench-verified based evaluation task. + + It takes the memory provided by the training agent and runs + all the selected SWE-bench-verified instances. Then provides + a combined score and feedback. + """ + + def __init__(self, config_file: Path) -> None: + super().__init__(config_file) + self.dataset: list[SweBenchInstance] = [] + self.parse_config(config_file=config_file) + + def repo_url(self) -> str: + """Return the URL of the repo for the training agent. + + Returns: + str: The URL of the repo for the training agent. + """ + return f"https://github.com/{self.dataset[0].repo}.git" + + def teardown(self, eval_repo_path: Path) -> None: + """Tear down the task, cleaning up any resources if necessary.""" + if eval_repo_path and eval_repo_path.exists(): + shutil.rmtree(eval_repo_path) + + def parse_config(self, config_file: Path) -> None: + """Parse the configuration file for the task. + The config file is a yaml file. It will have array of "instance_id" + or "repo" as the root object. Gather it and load the dataset to + the object variable dataset. + + Args: + config_file (Path): Path to the configuration file. + """ + + with open(config_file, "r") as f: + config = yaml.safe_load(f) + + instance_ids = config.get("instance_id_list", []) + repo = config.get("repo", None) + + if instance_ids: + repo = None + for instance_id in instance_ids: + dataset = load_instance_using_id(instance_id) + if not repo: + repo = dataset.repo + elif repo != dataset.repo: + raise ValueError( + f"Conflicting repos for instance_id {instance_id}: {repo} vs {dataset.repo}" + ) + + self.dataset.append(dataset) + + elif repo: + self.dataset = load_instances_of_repo(repo=repo) + + if len(self.dataset) == 0: + raise ValueError("No instances loaded for evaluation.") + + def eval(self, memory_dir: str, model: str, log_path: str) -> EvalOutcome: + """Runs the evaluation agent with the memory on all the eval instances + and produces a cumulative feedback. + + Args: + memory_dir (str): Path to the directory containing the agent's memory. + model (str): The model identifier used for evaluation. + log_path (str): Path to the log file for recording evaluation details. + + Returns: + EvalOutcome: The outcome of the evaluation, including whether it passed, the output, and the result. + """ + + eval_repo_path = Path(log_path).parent / "eval_repo" + results = [] + + for instance in self.dataset: + inst_log_path = Path(log_path).parent / f"{instance.instance_id}_log.txt" + task = SweBenchVerifiedTask_one(instance) + + res = task.eval(str(eval_repo_path), memory_dir, model, str(inst_log_path)) + + if not res.status: + logger.info(f"Evaluation failed for instance {instance.instance_id}: {res.error if res.error else 'Unknown error'}") + results.append(res) + else: + res = task.check(str(eval_repo_path), "", str(inst_log_path)) + results.append(res) + + score = 0 + for result in results: + if result.status: + score += 1 + + score = score / len(self.dataset) + + if score == 1: + feedback = "All evaluations passed." + else: + feedback = self._combine_result_feedback(results, model, str(eval_repo_path)) + + self.teardown(eval_repo_path) + + return EvalOutcome( + passed = score == 1, + score = score, + feedback = feedback + ) + + + def _combine_result_feedback(self, results: list[BotRunResult], model: str, eval_repo: str) -> str: + """ + Combines the feedback from multiple BotRunResult instances into a single feedback string. + Args: + results (list[BotRunResult]): List of individual bot run results. + model (str): The model identifier used for evaluation. + eval_repo (str): Path to the evaluation repository. + + Returns: + str: Combined feedback from all results. + """ + + serialized_str = f"Total {len(results)} tests ran and their result and feedback:\n" + + for res in results: + serialized_str += f"\nResult: {'Passed' if res.status else 'Failed'}\n" + serialized_str += f"Optional Feedback: {res.result if res.result else 'None'}\n" + serialized_str += f"Error if there are any: {res.error if res.error else 'None'}\n" + + try: + bot = ReadingBot( + model = model, + folder_to_mount=eval_repo + ) + task = f""" + Combine the results of the eval runs into single feedback. + This feedback will be given to the next iteration. + You just combine the results with minimal efforts. + Avoid referring to code whenever possible. + + {serialized_str} + """ + bot_result = bot.run(task=task) + except Exception as e: + logger.warning(f"Combining results failed with exception: {e}") + return f"Combining results failed. raw combined output:\n\n{serialized_str}" + + if bot_result.status: + return bot_result.result if bot_result.result else 'No feedback provided' + else: + return f"Combining results failed. raw combined output:\n\n{serialized_str}" diff --git a/src/microbots/auto_memory/evalTask.py b/src/microbots/auto_memory/evalTask.py index be3af99..fbe1290 100644 --- a/src/microbots/auto_memory/evalTask.py +++ b/src/microbots/auto_memory/evalTask.py @@ -7,52 +7,45 @@ from abc import ABC, abstractmethod from dataclasses import dataclass +from pathlib import Path from typing import Any + @dataclass -class CallbackResult: +class EvalOutcome: """Result of verifying whether an eval task was completed correctly. Attributes ---------- passed : bool Whether the agent's output satisfies the task's check. - reason : str + score : float + A numeric score representing the quality of the agent's output. + feedback : str A short human-readable explanation of the pass/fail verdict. """ passed: bool - reason: str - -@dataclass -class EvalOutcome: - """Full record of one eval round. - - Attributes - ---------- - passed : bool - Whether the round passed, mirrors ``result.passed``. - output : str | None - The agent's raw output for the round, if any. - result : CallbackResult - The verdict produced by ``EvalTask.check``. - """ - - passed: bool - output: str | None - result: CallbackResult - + score: float + feedback: str class EvalTask(ABC): """Base class for a single evaluation task in the train <-> eval loop. - Subclasses must implement ``run`` and ``from_config``. ``setup``, - ``build_prompt``, ``check``, and ``teardown`` are optional hooks + Subclasses must implement ``run``. ``parse_config``, ``setup``, + ``check``, and ``teardown`` are optional hooks subclasses may use to structure their own ``run`` implementation (see ``SweBenchVerifiedTask`` for an example), but nothing in this base class calls them automatically. """ + def __init__(self, config_file: Path) -> None: + super().__init__() + + @abstractmethod + def repo_url(self) -> str: + """Return the URL of the repo for the training agent.""" + @property def task_id(self) -> str: """Identifier for this task instance, used to name its output folder. @@ -69,26 +62,7 @@ def task_id(self) -> str: """ return type(self).__name__ - def build_result(self, outcome: EvalOutcome) -> dict: - """Optional. Build the dict written to this round's ``result.json``. - - Not called automatically; the orchestrator calls this after - each round to decide what to persist. Override to include - task-specific details (e.g. dataset fields, repo info). - - Parameters - ---------- - outcome : EvalOutcome - The round's outcome to summarize. - - Returns - ------- - dict - JSON-serializable summary. Defaults to ``passed``/``reason``. - """ - return {"passed": outcome.result.passed, "reason": outcome.result.reason} - - def setup(self, repo_path: str) -> None: + def setup(self) -> None: """Optional. Prepare repo/environment before the agent runs. Not called automatically; only useful if your ``run`` @@ -101,113 +75,34 @@ def setup(self, repo_path: str) -> None: """ pass - def build_prompt(self) -> str: - """Optional. Return the task prompt/instructions for the agent. - - Not called automatically; only useful if your ``run`` - implementation calls it. - - Returns - ------- - str - The prompt/instructions to give the agent. Empty string by - default. - """ - return "" - - def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackResult: - """Optional. Verify whether the task was actually completed correctly. - - Not called automatically; only useful if your ``run`` - implementation calls it. - - Parameters - ---------- - repo_path : str - Absolute path to the repo the agent operated on. - agent_output : str - The agent's raw output/result text. - log_path : str - Path to a log file, already created by ``run``, that this - check may append verification details to. - - Returns - ------- - CallbackResult - The pass/fail verdict and its reason. Passes by default. - """ - return CallbackResult(passed=True, reason="not checked") - - - def teardown(self, repo_path: str) -> None: + def teardown(self, eval_repo_path: Path) -> None: """Optional. Clean up anything setup() created. Parameters ---------- - repo_path : str + eval_repo_path : Path Absolute path to the repo that was prepared by ``setup``. """ pass - @classmethod - @abstractmethod - def from_config(cls, task_args: dict[str, Any]) -> list["EvalTask"]: - """Required. Build task instance(s) from a config's ``task_args`` dict. - - Parameters - ---------- - task_args : dict[str, Any] - Task-specific config values (the config file's - ``task_args`` section). - - Returns - ------- - list[EvalTask] - One task instance per unit of work this config describes - (often just one, but e.g. ``SweBenchVerifiedTask`` returns - one per matching dataset instance). - """ - raise NotImplementedError( - f"{cls.__name__} must implement from_config() to be usable via --task" - ) - @abstractmethod - def build_feedback(self, outcome: EvalOutcome, repo_path: str, model: str, log_path: str) -> str: - """Required. Analyze a failed eval outcome and produce training feedback. - - Called by the orchestrator after a failed round, before - retraining, to turn the round's outcome/log into concrete - feedback text describing what went wrong and what the agent's - memory notes should cover next time. + def parse_config(self, config_file: Path) -> None: + """Parse the task-specific config file. Importantly it + parses the config file and get the repo for the training + agent. Parameters ---------- - outcome : EvalOutcome - The failed outcome to analyze. - repo_path : str - Absolute path to the repo the task was evaluated against. - model : str - The model to use, in the format ``/``. - log_path : str - Path to the round's log file, containing the agent output - and any failure/exception details recorded during the - round (the same path passed to ``run``). - - Returns - ------- - str - Feedback text to pass as ``feedback`` to the next round's - training. + config_file : Path + Path to the config file to parse. """ @abstractmethod - def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> EvalOutcome: + def eval(self, memory_dir: str, model: str, log_path: str) -> EvalOutcome: """Required. Run one eval iteration and return its outcome. Parameters ---------- - repo_path : str - Absolute path to the repo to run the eval round against. memory_dir : str Directory containing memory files to give the agent via ``MemoryTool``. diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index e13ad3e..4691837 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -6,6 +6,7 @@ """ from dataclasses import dataclass, field +import dataclasses from logging import getLogger from pathlib import Path import json @@ -16,9 +17,7 @@ from microbots.auto_memory.training.runner import run_training from microbots.auto_memory.workdir import ( eval_log_path, - eval_repo_dir, eval_result_path, - load_config, load_round_memory, repo_dir, save_round_memory, @@ -87,10 +86,6 @@ def clone_repo(url: str, repo_path: Path) -> None: def write_eval_result(workdir: Path, round_num: int, task: EvalTask, outcome: EvalOutcome) -> None: """Write a round's eval result to ``result.json``. - Delegates the content to ``task.build_result(outcome)`` so each - task decides what's worth persisting (e.g. ``SweBenchVerifiedTask`` - includes its dataset instance's fields). - Parameters ---------- workdir : Path @@ -105,7 +100,7 @@ def write_eval_result(workdir: Path, round_num: int, task: EvalTask, outcome: Ev """ path = eval_result_path(workdir, round_num, task.task_id) path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(task.build_result(outcome), indent=2)) + path.write_text(json.dumps(dataclasses.asdict(outcome), indent=2)) def run_training_loop( repo_path: str, @@ -148,7 +143,6 @@ def run_training_loop( def run_train_eval_loop( training_repo_path: str, - eval_repo_path: str, workdir: Path, model: str, task: EvalTask, @@ -217,9 +211,10 @@ def run_train_eval_loop( logger.info( "run_train_eval_loop: round %d/%d starting", round_idx, max_rounds ) + # TODO: Instead of loading new memory dir on every iteration, snapshot the memory. memory_dir = str(load_round_memory(workdir, round_idx, instance_id=task.task_id)) log_path = str(eval_log_path(workdir, round_idx, task.task_id)) - outcome = task.run(eval_repo_path, memory_dir, model, log_path) + outcome = task.eval(memory_dir, model, log_path) outcomes.append(outcome) try: @@ -237,13 +232,12 @@ def run_train_eval_loop( logger.info( "run_train_eval_loop: round %d failed (%s), retraining", round_idx, - outcome.result.reason, + outcome.feedback, ) try: - feedback = task.build_feedback(outcome, eval_repo_path, model, log_path) run_training_loop( repo_path=training_repo_path, - feedback=feedback, + feedback=outcome.feedback, memory_dir=memory_dir, model=model, iterations=training_iterations, @@ -271,40 +265,31 @@ def run_train_eval_loop( def run( workdir: Path, model: str, - task: EvalTask | None, + task: EvalTask, max_rounds: int = 5, training_iterations: int = 10, - config: dict | None = None, -) -> LoopResult | None: - """Run training only, or the full train/eval loop, depending on ``task``. +) -> LoopResult: + """Run full train/eval loop, depending on ``task``. Parameters ---------- workdir : Path This run's workdir (see ``microbots.auto_memory.workdir``), - holding ``config.yaml``, the shared repo clone, and all output. + holding ``task_config.yaml``, the shared repo clone, and all output. model : str The model to use, in the format ``/``. - task : EvalTask | None - The eval task to run each round, or ``None`` to only run - training (with empty feedback, once per ``training_iterations``). + task : EvalTask + The eval task to run each round. max_rounds : int - Maximum number of train/eval rounds to attempt, if ``task`` is - given. Defaults to 5. + Maximum number of train/eval rounds to attempt. Defaults to 5. training_iterations : int Number of training passes to run per retraining round, each reusing the same round memory dir. Defaults to 10. - config : dict | None - This run's already-loaded ``config.yaml`` contents. If ``None`` - (the default), it is loaded from ``workdir`` here. Callers that - invoke ``run`` repeatedly for the same ``workdir`` (e.g. once - per eval task) can load it once and pass it in, to avoid - re-reading/re-parsing the file on every call. Returns ------- - LoopResult | None - The eval loop's result if ``task`` was given, otherwise ``None``. + LoopResult + The eval loop's result. Raises ------ @@ -315,38 +300,29 @@ def run( must be configured even for tasks like ``SweBenchVerifiedTask`` that manage their own separate eval checkout. """ - if config is None: - config = load_config(workdir) - repo_url = config.get("repo") - if not repo_url: - raise ValueError( - "config.yaml must specify 'repo' (the training checkout's clone " - "URL); it is required even when the eval task manages its own " - "separate eval repo checkout." - ) - clone_repo(repo_url, repo_dir(workdir)) + clone_repo(task.repo_url(), repo_dir(workdir)) snapshot_seed_memory(workdir) training_repo_path = str(repo_dir(workdir)) - if task is None: - # Train-only mode has no rounds of its own; round 1 is just a - # scratch dir seeded from (and saved back to) top-level memory. - memory_dir = str(load_round_memory(workdir, 1)) - run_training_loop( - repo_path=training_repo_path, - feedback="", - memory_dir=memory_dir, - model=model, - iterations=training_iterations, - ) - save_round_memory(workdir, 1) - return None + # TODO: train-only mode will be implemented if required after proper design + # if task is None: + # # Train-only mode has no rounds of its own; round 1 is just a + # # scratch dir seeded from (and saved back to) top-level memory. + # memory_dir = str(load_round_memory(workdir, 1)) + # run_training_loop( + # repo_path=training_repo_path, + # feedback="", + # memory_dir=memory_dir, + # model=model, + # iterations=training_iterations, + # ) + # save_round_memory(workdir, 1) + # return None return run_train_eval_loop( training_repo_path=training_repo_path, - eval_repo_path=str(eval_repo_dir(workdir)), workdir=workdir, model=model, task=task, diff --git a/src/microbots/auto_memory/task_registry.py b/src/microbots/auto_memory/task_registry.py index 63bfb74..1846d2d 100644 --- a/src/microbots/auto_memory/task_registry.py +++ b/src/microbots/auto_memory/task_registry.py @@ -7,14 +7,18 @@ import importlib import pkgutil +from collections.abc import Callable from microbots.auto_memory.evalTask import EvalTask TASK_REGISTRY: dict[str, type[EvalTask]] = {} -def register_task(name: str): +def register_task(name: str) -> Callable[[type[EvalTask]], type[EvalTask]]: """Register an ``EvalTask`` subclass under ``name`` as a class decorator. + Each name maps to exactly one class; registering a name twice is a + programming error rather than a silent overwrite. + Parameters ---------- name : str @@ -40,27 +44,39 @@ def decorator(task_cls: type[EvalTask]) -> type[EvalTask]: ------- type[EvalTask] ``task_cls``, unchanged. + + Raises + ------ + ValueError + If ``name`` is already registered to a different class. """ + registered = TASK_REGISTRY.get(name) + if registered is not None and registered is not task_cls: + raise ValueError( + f"Task name {name!r} is already registered to " + f"{registered.__module__}.{registered.__qualname__}; " + f"cannot also register {task_cls.__module__}.{task_cls.__qualname__}." + ) TASK_REGISTRY[name] = task_cls return task_cls return decorator -# Not being used currently, but kept it for future use if required. -def create_task(name: str, **kwargs) -> EvalTask: - """Construct a registered ``EvalTask`` by name. +def create_task(name: str) -> EvalTask: + """Construct the registered ``EvalTask`` for ``name``. + + Tasks take no constructor arguments; per-run configuration is + applied afterwards via ``EvalTask.parse_config``. Parameters ---------- name : str The registered task name, e.g. ``"swebenchverified"``. - **kwargs - Keyword arguments forwarded to the task's constructor. Returns ------- EvalTask - The constructed task instance. + A new instance of the class registered under ``name``. Raises ------ @@ -73,7 +89,7 @@ def create_task(name: str, **kwargs) -> EvalTask: raise ValueError( f"Unknown task {name!r}. Registered tasks: {sorted(TASK_REGISTRY)}" ) from None - return task_cls(**kwargs) + return task_cls() def discover_tasks(package_name: str = "microbots.auto_memory.eval") -> None: """Import every module in ``package_name`` so ``@register_task`` fires. diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py index 154699b..1276ff2 100644 --- a/src/microbots/auto_memory/workdir.py +++ b/src/microbots/auto_memory/workdir.py @@ -38,24 +38,24 @@ def resolve_workdir(base: Path | None = None) -> Path: Path ``workdir`` resolved relative to ``base`` (or ``Path.cwd()``). """ - return (base or Path.cwd()) / WORKDIR_NAME + if base is not None and not base.is_absolute(): + raise ValueError(f"base must be an absolute path: {base}") + workdir = (base or Path.cwd()) / WORKDIR_NAME + return workdir def require_workdir(workdir: Path) -> None: - """Validate that ``workdir`` exist. + """Validate that ``workdir`` exist. Create if not exist Parameters ---------- workdir : Path The workdir to validate. - Raises - ------ - FileNotFoundError - If ``workdir`` does not exist. """ - if not workdir.is_dir(): - raise FileNotFoundError(f"workdir not found: {workdir}") + # create workdir if not existing + if not workdir.exists(): + workdir.mkdir(parents=True) def config_path(workdir: Path) -> Path: @@ -117,28 +117,6 @@ def repo_dir(workdir: Path) -> Path: return workdir / REPO_DIRNAME -def eval_repo_dir(workdir: Path) -> Path: - """Return the path to the repo an eval task clones/manages itself. - - Kept separate from ``repo_dir`` (the training repo) because a - task's ``setup`` may clone or reset this directory every round - (e.g. ``SweBenchVerifiedTask`` checks out a different repo/commit - per dataset instance), which would otherwise conflict with the - persistent training checkout at ``repo_dir``. - - Parameters - ---------- - workdir : Path - The run's workdir. - - Returns - ------- - Path - ``workdir/eval_repo``. - """ - return workdir / EVAL_REPO_DIRNAME - - def memory_dir(workdir: Path) -> Path: """Return the path to the current top-level (latest) memory directory. diff --git a/src/microbots/auto_memory/workdir/config.yaml b/src/microbots/auto_memory/workdir/config.yaml new file mode 100644 index 0000000..03de1e5 --- /dev/null +++ b/src/microbots/auto_memory/workdir/config.yaml @@ -0,0 +1,3 @@ +instance_id_list: + - astropy__astropy-12907 + - astropy__astropy-13033 \ No newline at end of file From 89053900aa7e4021da99d9d3b1a52ef3032a5cf8 Mon Sep 17 00:00:00 2001 From: bala Date: Tue, 8 Sep 2026 09:39:23 +0000 Subject: [PATCH 14/21] Clean and Prune code to make it simple --- src/microbots/auto_memory/cli.py | 11 +- .../auto_memory/eval/swebenchverified.py | 20 +- src/microbots/auto_memory/evalTask.py | 61 +--- src/microbots/auto_memory/orchestrator.py | 122 +++---- src/microbots/auto_memory/task_registry.py | 28 -- src/microbots/auto_memory/training/runner.py | 10 +- src/microbots/auto_memory/workdir.py | 333 +++++------------- 7 files changed, 158 insertions(+), 427 deletions(-) diff --git a/src/microbots/auto_memory/cli.py b/src/microbots/auto_memory/cli.py index 383f313..f4ba0da 100644 --- a/src/microbots/auto_memory/cli.py +++ b/src/microbots/auto_memory/cli.py @@ -54,12 +54,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "--config-file", type=Path, help="Path to the task configuration file.", ) parser.add_argument("--max-rounds", type=int, default=5) - parser.add_argument("--training-iterations", type=int, default=10) return parser.parse_args(argv) def main(argv: list[str] | None = None) -> None: - """CLI entry point: run training only, or the full train/eval loop. + """CLI entry point: run the full train/eval loop. Parameters ---------- @@ -68,6 +67,13 @@ def main(argv: list[str] | None = None) -> None: """ args = parse_args(argv) + # The user can pass either an existing workdir containing a + # task_config.yml file or a task_config.yml file using --config + # option. In the later case, the workdir will be created in + # the default location. + # If both workdir and config options are provided and there exists + # a task_yaml.yml inside the workdir, that file will be ignored and + # the provided --config-file will take precedence. workdir = Path(args.workdir) if args.workdir else resolve_workdir() require_workdir(workdir) @@ -83,7 +89,6 @@ def main(argv: list[str] | None = None) -> None: model=args.model, task=TASK_REGISTRY[args.task](config_file=config_file), max_rounds=args.max_rounds, - training_iterations=args.training_iterations, ) if result is not None: logger.info( diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index 31d2866..a840170 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -395,7 +395,6 @@ def eval(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Bo error=f"{type(exc).__name__}: {exc}" ) - @register_task("swebenchverified") class SweBenchVerified(EvalTask): """SWE-bench-verified based evaluation task. @@ -406,7 +405,7 @@ class SweBenchVerified(EvalTask): """ def __init__(self, config_file: Path) -> None: - super().__init__(config_file) + # No need to call the base-class init self.dataset: list[SweBenchInstance] = [] self.parse_config(config_file=config_file) @@ -418,11 +417,6 @@ def repo_url(self) -> str: """ return f"https://github.com/{self.dataset[0].repo}.git" - def teardown(self, eval_repo_path: Path) -> None: - """Tear down the task, cleaning up any resources if necessary.""" - if eval_repo_path and eval_repo_path.exists(): - shutil.rmtree(eval_repo_path) - def parse_config(self, config_file: Path) -> None: """Parse the configuration file for the task. The config file is a yaml file. It will have array of "instance_id" @@ -458,7 +452,7 @@ def parse_config(self, config_file: Path) -> None: if len(self.dataset) == 0: raise ValueError("No instances loaded for evaluation.") - def eval(self, memory_dir: str, model: str, log_path: str) -> EvalOutcome: + def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: """Runs the evaluation agent with the memory on all the eval instances and produces a cumulative feedback. @@ -470,12 +464,13 @@ def eval(self, memory_dir: str, model: str, log_path: str) -> EvalOutcome: Returns: EvalOutcome: The outcome of the evaluation, including whether it passed, the output, and the result. """ - - eval_repo_path = Path(log_path).parent / "eval_repo" + eval_path = Path(eval_dir) + eval_repo_path = eval_path / "eval_repo" + eval_log_dir = eval_path / "logs" results = [] for instance in self.dataset: - inst_log_path = Path(log_path).parent / f"{instance.instance_id}_log.txt" + inst_log_path = eval_log_dir / f"{instance.instance_id}_log.txt" task = SweBenchVerifiedTask_one(instance) res = task.eval(str(eval_repo_path), memory_dir, model, str(inst_log_path)) @@ -499,7 +494,7 @@ def eval(self, memory_dir: str, model: str, log_path: str) -> EvalOutcome: else: feedback = self._combine_result_feedback(results, model, str(eval_repo_path)) - self.teardown(eval_repo_path) + # NOTE: Let's not teardown the repository as it will be useful for debugging return EvalOutcome( passed = score == 1, @@ -507,7 +502,6 @@ def eval(self, memory_dir: str, model: str, log_path: str) -> EvalOutcome: feedback = feedback ) - def _combine_result_feedback(self, results: list[BotRunResult], model: str, eval_repo: str) -> str: """ Combines the feedback from multiple BotRunResult instances into a single feedback string. diff --git a/src/microbots/auto_memory/evalTask.py b/src/microbots/auto_memory/evalTask.py index fbe1290..dd9762b 100644 --- a/src/microbots/auto_memory/evalTask.py +++ b/src/microbots/auto_memory/evalTask.py @@ -8,7 +8,8 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from pathlib import Path -from typing import Any + +import yaml @dataclass @@ -41,51 +42,15 @@ class EvalTask(ABC): def __init__(self, config_file: Path) -> None: super().__init__() + self.parse_config(config_file=config_file) - @abstractmethod def repo_url(self) -> str: """Return the URL of the repo for the training agent.""" + if not self._repo_url: + raise ValueError("Repo URL is not set in the config file." + " Or you didn't override the base method.") + return self._repo_url - @property - def task_id(self) -> str: - """Identifier for this task instance, used to name its output folder. - - Defaults to the class name, which is fine for tasks with only - one instance per run. Override for tasks with several distinct - instances per class (e.g. ``SweBenchVerifiedTask``, where each - dataset row needs its own folder). - - Returns - ------- - str - This task instance's identifier. - """ - return type(self).__name__ - - def setup(self) -> None: - """Optional. Prepare repo/environment before the agent runs. - - Not called automatically; only useful if your ``run`` - implementation calls it. - - Parameters - ---------- - repo_path : str - Absolute path to the repo to prepare. - """ - pass - - def teardown(self, eval_repo_path: Path) -> None: - """Optional. Clean up anything setup() created. - - Parameters - ---------- - eval_repo_path : Path - Absolute path to the repo that was prepared by ``setup``. - """ - pass - - @abstractmethod def parse_config(self, config_file: Path) -> None: """Parse the task-specific config file. Importantly it parses the config file and get the repo for the training @@ -96,9 +61,12 @@ def parse_config(self, config_file: Path) -> None: config_file : Path Path to the config file to parse. """ + with open(config_file, "r") as f: + config = yaml.safe_load(f) + self._repo_url = config.get("repo") @abstractmethod - def eval(self, memory_dir: str, model: str, log_path: str) -> EvalOutcome: + def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: """Required. Run one eval iteration and return its outcome. Parameters @@ -108,10 +76,9 @@ def eval(self, memory_dir: str, model: str, log_path: str) -> EvalOutcome: ``MemoryTool``. model : str The model to use, in the format ``/``. - log_path : str - Path to write this round's log to. Caller-provided (e.g. a - workdir-managed path) so logs persist under the run's - layout instead of each task inventing its own temp file. + eval_dir: str + Path to run this round's eval. This directory is managed by + the eval task itself. It can have its cloned repo, logs, etc. Returns ------- diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index 4691837..78d7754 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -5,23 +5,22 @@ passes or ``max_rounds`` is exhausted. """ -from dataclasses import dataclass, field import dataclasses -from logging import getLogger -from pathlib import Path import json import shutil import subprocess +from dataclasses import dataclass, field +from logging import getLogger +from pathlib import Path from microbots.auto_memory.evalTask import EvalOutcome, EvalTask from microbots.auto_memory.training.runner import run_training from microbots.auto_memory.workdir import ( - eval_log_path, - eval_result_path, - load_round_memory, + RESULT_FILENAME, + get_eval_dir, + memory_dir, repo_dir, - save_round_memory, - snapshot_seed_memory, + take_memory_snapshot, ) logger = getLogger(__name__) @@ -83,63 +82,20 @@ def clone_repo(url: str, repo_path: Path) -> None: subprocess.run(["git", "clone", url, str(repo_path)], check=True) -def write_eval_result(workdir: Path, round_num: int, task: EvalTask, outcome: EvalOutcome) -> None: +def write_eval_result(eval_dir: Path, outcome: EvalOutcome) -> None: """Write a round's eval result to ``result.json``. Parameters ---------- - workdir : Path - The run's workdir. - round_num : int - 1-based round number this outcome belongs to. - task : EvalTask - The task that produced ``outcome``, used for both its - ``task_id`` (folder name) and ``build_result`` (file content). + eval_dir : Path + The directory for this round's evaluation. outcome : EvalOutcome The round's outcome to persist. """ - path = eval_result_path(workdir, round_num, task.task_id) + path = eval_dir / RESULT_FILENAME path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(dataclasses.asdict(outcome), indent=2)) -def run_training_loop( - repo_path: str, - feedback: str, - memory_dir: str, - model: str, - iterations: int = 10, -) -> None: - """Run ``run_training`` ``iterations`` times, reusing the same memory dir. - - Shared by the eval-loop's retrain step and any training-only entry - point (e.g. a CLI) that needs to run training without an eval task. - - Parameters - ---------- - repo_path : str - Absolute path to the repo to train against. - feedback : str - Feedback from a prior failed eval attempt, or ``""`` if none. - memory_dir : str - Directory where the training agent reads/writes memory files. - model : str - The model to use, in the format ``/``. - iterations : int - Number of training passes to run, each reusing the same - ``memory_dir``. Defaults to 10. - """ - for iteration in range(1, iterations + 1): - logger.info( - "run_training_loop: training iteration %d/%d", - iteration, - iterations, - ) - run_training( - repo_path=repo_path, - feedback=feedback, - memory_dir=memory_dir, - model=model, - ) def run_train_eval_loop( training_repo_path: str, @@ -147,7 +103,6 @@ def run_train_eval_loop( model: str, task: EvalTask, max_rounds: int = 5, - training_iterations: int = 10, ) -> LoopResult: """Run an eval task in a loop, retraining on failure until it passes. @@ -174,10 +129,9 @@ def run_train_eval_loop( retraining (``run_training_loop``). Kept separate from ``eval_repo_path`` since the task manages the latter's lifecycle itself (clone/teardown each round). - eval_repo_path : str - Absolute path to the repo the task clones/manages itself (via - its own ``setup``) and runs/checks the agent against each - round. + training_repo_path : str + Absolute path to the persistent repo checkout used only for + retraining (``run_training_loop``). workdir : Path This run's workdir, used to carry memory forward between rounds (see ``microbots.auto_memory.workdir``). @@ -187,9 +141,6 @@ def run_train_eval_loop( The eval task to run each round. max_rounds : int Maximum number of train/eval rounds to attempt. Defaults to 5. - training_iterations : int - Number of training passes to run per retraining round, each - reusing the same round memory dir. Defaults to 10. Returns ------- @@ -207,15 +158,27 @@ def run_train_eval_loop( outcomes: list[EvalOutcome] = [] + #TODO: Logs need to be saved to appropriate log files for round_idx in range(1, max_rounds+1): logger.info( "run_train_eval_loop: round %d/%d starting", round_idx, max_rounds ) - # TODO: Instead of loading new memory dir on every iteration, snapshot the memory. - memory_dir = str(load_round_memory(workdir, round_idx, instance_id=task.task_id)) - log_path = str(eval_log_path(workdir, round_idx, task.task_id)) - outcome = task.eval(memory_dir, model, log_path) - outcomes.append(outcome) + mem_dir = memory_dir(Path(workdir)) + eval_dir = get_eval_dir(workdir, round_idx) + take_memory_snapshot(mem_dir, round_idx) + try: + outcome = task.eval(str(mem_dir), model, str(eval_dir)) + outcomes.append(outcome) + except Exception as e: + logger.warning( + "run_train_eval_loop: round %d failed during evaluation;\n" + "Exception: %s\n" + "continuing to next round", + round_idx, e + ) + # Store the failure in outcomes + outcomes.append(EvalOutcome(passed=False, score=-1, feedback=str(e))) + continue try: if outcome.passed: @@ -234,14 +197,14 @@ def run_train_eval_loop( round_idx, outcome.feedback, ) + try: - run_training_loop( + run_training( repo_path=training_repo_path, feedback=outcome.feedback, - memory_dir=memory_dir, + memory_dir=str(mem_dir), model=model, - iterations=training_iterations, - ) + ) except Exception: logger.exception( "run_train_eval_loop: round %d failed to build feedback/retrain; " @@ -249,8 +212,7 @@ def run_train_eval_loop( round_idx, ) finally: - write_eval_result(workdir, round_idx, task, outcome) - save_round_memory(workdir, round_idx, instance_id=task.task_id) + write_eval_result(eval_dir, outcome) logger.info( "run_train_eval_loop: exhausted %d rounds without passing", max_rounds @@ -267,7 +229,6 @@ def run( model: str, task: EvalTask, max_rounds: int = 5, - training_iterations: int = 10, ) -> LoopResult: """Run full train/eval loop, depending on ``task``. @@ -300,11 +261,9 @@ def run( must be configured even for tasks like ``SweBenchVerifiedTask`` that manage their own separate eval checkout. """ - clone_repo(task.repo_url(), repo_dir(workdir)) - - snapshot_seed_memory(workdir) - - training_repo_path = str(repo_dir(workdir)) + training_repo_dir = repo_dir(workdir) + clone_repo(task.repo_url(), training_repo_dir) + training_repo_path = str(training_repo_dir) # TODO: train-only mode will be implemented if required after proper design # if task is None: @@ -327,5 +286,4 @@ def run( model=model, task=task, max_rounds=max_rounds, - training_iterations=training_iterations, - ) + ) \ No newline at end of file diff --git a/src/microbots/auto_memory/task_registry.py b/src/microbots/auto_memory/task_registry.py index 1846d2d..f8720ae 100644 --- a/src/microbots/auto_memory/task_registry.py +++ b/src/microbots/auto_memory/task_registry.py @@ -62,34 +62,6 @@ def decorator(task_cls: type[EvalTask]) -> type[EvalTask]: return decorator -def create_task(name: str) -> EvalTask: - """Construct the registered ``EvalTask`` for ``name``. - - Tasks take no constructor arguments; per-run configuration is - applied afterwards via ``EvalTask.parse_config``. - - Parameters - ---------- - name : str - The registered task name, e.g. ``"swebenchverified"``. - - Returns - ------- - EvalTask - A new instance of the class registered under ``name``. - - Raises - ------ - ValueError - If ``name`` has not been registered via ``register_task``. - """ - try: - task_cls = TASK_REGISTRY[name] - except KeyError: - raise ValueError( - f"Unknown task {name!r}. Registered tasks: {sorted(TASK_REGISTRY)}" - ) from None - return task_cls() def discover_tasks(package_name: str = "microbots.auto_memory.eval") -> None: """Import every module in ``package_name`` so ``@register_task`` fires. diff --git a/src/microbots/auto_memory/training/runner.py b/src/microbots/auto_memory/training/runner.py index 94f6700..5401f5f 100644 --- a/src/microbots/auto_memory/training/runner.py +++ b/src/microbots/auto_memory/training/runner.py @@ -3,8 +3,8 @@ from pathlib import Path from microbots.bot.ReadingBot import ReadingBot -from microbots.tools.tool_definitions.memory_tool import MemoryTool from microbots.MicroBot import BotRunResult +from microbots.tools.tool_definitions.memory_tool import MemoryTool _INSTRUCTIONS_PATH = Path(__file__).parent / "training_instructions.md" @@ -14,8 +14,8 @@ def run_training( feedback: str, memory_dir: str, model: str, - max_iterations: int = 20, - timeout_in_seconds: int = 600, + max_iterations: int = 200, + timeout_in_seconds: int = 3600, ) -> BotRunResult: """Run one training pass over a repository and update its memory. @@ -33,9 +33,9 @@ def run_training( Directory in which the memory tool stores its memory. model : str Model identifier used by the reading bot. - max_iterations : int, default=20 + max_iterations : int, default=200 Maximum number of bot iterations. - timeout_in_seconds : int, default=600 + timeout_in_seconds : int, default=3600 Maximum duration of the bot run in seconds. Returns diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py index 1276ff2..2f7693c 100644 --- a/src/microbots/auto_memory/workdir.py +++ b/src/microbots/auto_memory/workdir.py @@ -5,23 +5,43 @@ outputs), so callers never hard-code layout details themselves. """ -from pathlib import Path +import os import shutil - -import yaml +import time +from pathlib import Path WORKDIR_NAME = "workdir" -CONFIG_FILENAME = "config.yaml" +CONFIG_FILENAME = "task_config.yaml" REPO_DIRNAME = "repo" -EVAL_REPO_DIRNAME = "eval_repo" MEMORY_DIRNAME = "memory" -MEMORY_SEED_DIRNAME = "memory_seed" +""" +At the beginning of the round, memory from memory_dir will be snapshotted +inside the workdir/rounds/rounds_n/starting_memory_snapshot directory. +So, the final memory after that round should be found at rounds_(n+1) +directory. The memory of the final round will be available at the +main memory dir workdir/memory +""" +STARTING_MEMORY_SNAPSHOT_DIR = "starting_memory_snapshot" ROUNDS_DIRNAME = "rounds" -ROUND_LOG_FILENAME = "round.log" -ROUND_PATCH_FILENAME = "repo.patch" +ROUND_LOG_DIR= "logs" EVAL_DIRNAME = "eval" RESULT_FILENAME = "result.json" -EVAL_LOG_FILENAME = "eval.log" + +""" +Expected workdir structure: + + workdir/ + | + workdir/ + ├── task_config.yaml + ├── repo/ + ├── memory/ + ├── rounds/round_n/ + │ └── logs/ <-- Contains only training log. Eval logs can be found inside the eval task + │ └── eval/ <-- Managed by the eval task + │ └── starting_memory_snapshot/ + └── logs/ +""" def resolve_workdir(base: Path | None = None) -> Path: @@ -38,8 +58,6 @@ def resolve_workdir(base: Path | None = None) -> Path: Path ``workdir`` resolved relative to ``base`` (or ``Path.cwd()``). """ - if base is not None and not base.is_absolute(): - raise ValueError(f"base must be an absolute path: {base}") workdir = (base or Path.cwd()) / WORKDIR_NAME return workdir @@ -55,7 +73,30 @@ def require_workdir(workdir: Path) -> None: """ # create workdir if not existing if not workdir.exists(): - workdir.mkdir(parents=True) + mem_dir = workdir / MEMORY_DIRNAME + mem_dir.mkdir(parents=True) + return + + workdir_parent = workdir.parent + workdir_backup = workdir_parent / f"{workdir.name}_backup_{int(time.time())}" + shutil.move(str(workdir), str(workdir_backup)) + workdir.mkdir(parents=True) + + task_config = workdir_backup / CONFIG_FILENAME + if task_config.exists(): + shutil.copy(task_config, workdir / CONFIG_FILENAME) + else: + raise FileNotFoundError(f"Task config file does not exist: {task_config}") + + mem_dir = workdir_backup / MEMORY_DIRNAME + if mem_dir.exists(): + shutil.copytree(mem_dir, workdir / MEMORY_DIRNAME) + else: + mem_dir.mkdir(parents=True) + + repo_dir = workdir_backup / REPO_DIRNAME + if repo_dir.exists(): + shutil.copytree(repo_dir, workdir / REPO_DIRNAME) def config_path(workdir: Path) -> Path: @@ -74,26 +115,6 @@ def config_path(workdir: Path) -> Path: return workdir / CONFIG_FILENAME -def load_config(workdir: Path) -> dict: - """Load and parse ``workdir``'s config file. - - Parameters - ---------- - workdir : Path - The workdir whose config file should be loaded. - - Returns - ------- - dict - The parsed config, or ``{}`` if the config file doesn't exist - or is empty. - """ - path = config_path(workdir) - if not path.is_file(): - return {} - return yaml.safe_load(path.read_text()) or {} - - def repo_dir(workdir: Path) -> Path: """Return the path to the single cloned repo shared across rounds. @@ -123,7 +144,7 @@ def memory_dir(workdir: Path) -> Path: Parameters ---------- workdir : Path - The run's workdir. + The run's workdir. Returns ------- @@ -133,46 +154,30 @@ def memory_dir(workdir: Path) -> Path: return workdir / MEMORY_DIRNAME -def snapshot_seed_memory(workdir: Path) -> Path: - """Snapshot the current top-level memory dir as the run's restorable baseline. - - ``memory_dir`` is shared and mutated in place across every - training/eval round and every eval task instance (so later - instances benefit from what earlier ones learned), which means the - original, pre-run memory is otherwise overwritten and lost with no - way to get back to it. Call this once, before anything trains, - to preserve that original state at ``workdir/memory_seed``. A - no-op if a snapshot already exists, so later calls (e.g. once per - eval task instance in the same run) never clobber the very first - snapshot with already-mutated memory. +def take_memory_snapshot(mem_dir: Path, round_idx: int) -> None: + """Take a snapshot of the current memory directory for a specific round. Parameters ---------- - workdir : Path - The run's workdir. - - Returns - ------- - Path - ``workdir/memory_seed``, containing a copy of whatever - ``memory_dir`` held the first time this was called (or empty, - if there was no pre-existing memory). + mem_dir : Path + The path to the current top-level memory directory. + round_idx : int + The 1-based round number. """ - dst = workdir / MEMORY_SEED_DIRNAME - if dst.exists(): - return dst - src = memory_dir(workdir) - if src.is_dir(): - shutil.copytree(src, dst) - else: - dst.mkdir(parents=True, exist_ok=True) - return dst + if not mem_dir.exists(): + # At this point, mem_dir can be empty but it should exist + raise FileNotFoundError(f"Memory directory does not exist: {mem_dir}") + + workdir = Path(mem_dir).parent + snapshot_dir = round_dir(workdir, round_idx) / STARTING_MEMORY_SNAPSHOT_DIR + if os.path.exists(snapshot_dir): + shutil.rmtree(snapshot_dir) + shutil.copytree(mem_dir, snapshot_dir) def round_dir( - workdir: Path, round_num: int, *, instance_id: str | None = None, create: bool = False -) -> Path: - """Return (and optionally create) the directory for a training round. + workdir: Path, round_num: int) -> Path: + """Create and return the directory for a training round. Parameters ---------- @@ -180,120 +185,18 @@ def round_dir( The run's workdir. round_num : int 1-based round number. - instance_id : str | None - If given, rounds are kept under a per-instance rounds dir - (``rounds_{instance_id}``) instead of the shared ``rounds`` dir, - so different eval task instances sharing the same ``workdir`` - don't collide on round numbers. Pass the eval task's - ``task_id`` when running an eval task; omit for training-only - mode. - create : bool - If True, create the directory (and parents) if missing. Returns ------- Path - ``workdir/rounds/round_{round_num}`` (no ``instance_id``), or - ``workdir/rounds_{instance_id}/round_{round_num}``. + ``workdir/rounds/round_{round_num}`` """ - rounds_dirname = f"{ROUNDS_DIRNAME}_{instance_id}" if instance_id else ROUNDS_DIRNAME - path = workdir / rounds_dirname / f"round_{round_num}" - if create: - path.mkdir(parents=True, exist_ok=True) + path = workdir / ROUNDS_DIRNAME / f"round_{round_num}" + path.mkdir(parents=True, exist_ok=True) return path -def round_memory_dir(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: - """Return the path to a round's own memory snapshot (a directory). - - Parameters - ---------- - workdir : Path - The run's workdir. - round_num : int - 1-based round number. - instance_id : str | None - The eval task's ``task_id``, if running an eval task (see - ``round_dir``). Omit for training-only mode. - - Returns - ------- - Path - This round's own memory directory. - """ - return round_dir(workdir, round_num, instance_id=instance_id) / MEMORY_DIRNAME - - -def load_round_memory(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: - """Copy the current top-level memory into this round's own memory dir. - - Called before a round's training pass, so it starts from whatever - memory the previous round left behind (or empty, on round 1). This - round's memory dir is replaced, not merged into: any stale files - left behind by a previous attempt at this same round (e.g. a - crashed/re-run process) are discarded first, so the round always - starts from an exact snapshot of the current top-level memory. - - Parameters - ---------- - workdir : Path - The run's workdir. - round_num : int - 1-based round number to load memory into. - instance_id : str | None - The eval task's ``task_id``, if running an eval task (see - ``round_dir``). Omit for training-only mode. - - Returns - ------- - Path - This round's own memory dir, ready for the round to use. - """ - src = memory_dir(workdir) - dst = round_memory_dir(workdir, round_num, instance_id=instance_id) - shutil.rmtree(dst, ignore_errors=True) - if src.is_dir(): - shutil.copytree(src, dst) - else: - dst.mkdir(parents=True, exist_ok=True) - return dst - - -def save_round_memory(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: - """Copy this round's memory back up to the top-level memory dir. - - Called after a round's training pass, so later rounds (and the - final saved memory) see what this round learned. The top-level - memory dir is replaced, not merged into: files the round deleted - (e.g. via the agent's ``memory delete`` command) are gone from - the top level too, instead of surviving from a previous save. - - Parameters - ---------- - workdir : Path - The run's workdir. - round_num : int - 1-based round number whose memory should be saved. - instance_id : str | None - The eval task's ``task_id``, if running an eval task (see - ``round_dir``). Omit for training-only mode. - - Returns - ------- - Path - The top-level ``memory`` dir, now updated with this round's changes. - """ - src = round_memory_dir(workdir, round_num, instance_id=instance_id) - dst = memory_dir(workdir) - shutil.rmtree(dst, ignore_errors=True) - if src.is_dir(): - shutil.copytree(src, dst) - else: - dst.mkdir(parents=True, exist_ok=True) - return dst - - -def round_log_path(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: +def round_log_dir(workdir: Path, round_num: int) -> Path: """Return the path to a round's training log. Parameters @@ -302,22 +205,19 @@ def round_log_path(workdir: Path, round_num: int, *, instance_id: str | None = N The run's workdir. round_num : int 1-based round number. - instance_id : str | None - The eval task's ``task_id``, if running an eval task (see - ``round_dir``). Omit for training-only mode. Returns ------- Path - This round's ``round.log``. + This round's log directory. Both training and eval logs + can be found inside with appropriate file names """ - return round_dir(workdir, round_num, instance_id=instance_id) / ROUND_LOG_FILENAME + return round_dir(workdir, round_num) / ROUND_LOG_DIR -def eval_dir( - workdir: Path, round_num: int, instance_id: str, *, create: bool = False -) -> Path: - """Return (and optionally create) an eval task instance's eval directory. +def get_eval_dir( + workdir: Path, round_num: int) -> Path: + """Return the eval task instance's eval directory. Creates it if missing. Parameters ---------- @@ -325,77 +225,12 @@ def eval_dir( The run's workdir. round_num : int 1-based round number this eval instance belongs to. - instance_id : str - The eval task instance identifier. - create : bool - If True, create the directory (and parents) if missing. Returns ------- Path - ``workdir/rounds_{instance_id}/round_{round_num}/eval``. + ``workdir/rounds/round_{round_num}/eval``. """ - path = round_dir(workdir, round_num, instance_id=instance_id) / EVAL_DIRNAME - if create: - path.mkdir(parents=True, exist_ok=True) + path = round_dir(workdir, round_num) / EVAL_DIRNAME + path.mkdir(parents=True, exist_ok=True) return path - - -def eval_result_path(workdir: Path, round_num: int, instance_id: str) -> Path: - """Return the path to an eval instance's result file. - - Parameters - ---------- - workdir : Path - The run's workdir. - round_num : int - 1-based round number this eval instance belongs to. - instance_id : str - The eval task instance identifier. - - Returns - ------- - Path - This eval instance's ``result.json``. - """ - return eval_dir(workdir, round_num, instance_id) / RESULT_FILENAME - - -def eval_log_path(workdir: Path, round_num: int, instance_id: str) -> Path: - """Return the path to an eval instance's log file. - - Parameters - ---------- - workdir : Path - The run's workdir. - round_num : int - 1-based round number this eval instance belongs to. - instance_id : str - The eval task instance identifier. - - Returns - ------- - Path - This eval instance's ``eval.log``. - """ - return eval_dir(workdir, round_num, instance_id) / EVAL_LOG_FILENAME - - -def eval_patch_path(workdir: Path, round_num: int, instance_id: str) -> Path: - """Return the path to an eval instance's captured repo diff. - - Parameters - ---------- - workdir : Path - The run's workdir. - round_num : int - 1-based round number this eval instance belongs to. - instance_id : str - The eval task instance identifier. - - Returns - ------- - Path - This eval instance's ``repo.patch``. - """ - return eval_dir(workdir, round_num, instance_id) / ROUND_PATCH_FILENAME \ No newline at end of file From fdef49ac4999fce73ca46ac3720dbd3741204678 Mon Sep 17 00:00:00 2001 From: bala Date: Tue, 8 Sep 2026 10:31:53 +0000 Subject: [PATCH 15/21] Clean docstrings, fix minor bugs and rewrite test cases --- src/microbots/auto_memory/architecture.md | 206 ++--- src/microbots/auto_memory/cli.py | 4 +- src/microbots/auto_memory/eval/__init__.py | 1 + .../auto_memory/eval/swebenchverified.py | 132 ++-- src/microbots/auto_memory/evalTask.py | 60 +- src/microbots/auto_memory/orchestrator.py | 59 +- src/microbots/auto_memory/task_registry.py | 9 +- src/microbots/auto_memory/workdir.py | 102 ++- .../auto_memory/eval/test_swebenchverified.py | 729 ++++-------------- test/auto_memory/test_cli.py | 159 ++-- test/auto_memory/test_full_loop.py | 129 ++++ test/auto_memory/test_orchestrator.py | 529 ++----------- test/auto_memory/test_task.py | 117 --- test/auto_memory/test_task_registry.py | 104 +-- test/auto_memory/test_workdir.py | 288 ++----- 15 files changed, 738 insertions(+), 1890 deletions(-) create mode 100644 src/microbots/auto_memory/eval/__init__.py create mode 100644 test/auto_memory/test_full_loop.py delete mode 100644 test/auto_memory/test_task.py diff --git a/src/microbots/auto_memory/architecture.md b/src/microbots/auto_memory/architecture.md index 01c9135..12c2418 100644 --- a/src/microbots/auto_memory/architecture.md +++ b/src/microbots/auto_memory/architecture.md @@ -1,9 +1,9 @@ # auto_memory — Architecture An agent that **learns a repository into memory notes**, then **proves those notes work** by -solving a real task with them. If it fails, it learns again from the failure and retries. +solving real SWE-bench issues with them. If it fails, it learns from the failure and retries. -> Train → Eval → Feedback → Train → … until pass (or rounds run out). +> Train → Eval → Feedback → Train → … until every instance passes (or rounds run out). --- @@ -11,23 +11,23 @@ solving a real task with them. If it fails, it learns again from the failure and ```mermaid flowchart LR - subgraph LOOP["Train / Eval Loop"] + CLI["cli.py
--model --task --config-file --max-rounds"] --> LOOP + + subgraph LOOP["orchestrator: train / eval loop"] direction TB - T["🧠 TRAIN
ReadingBot reads the repo
writes notes to memory/"] - E["🎯 EVAL
WritingBot solves a task
using only those notes"] - C{"Passed?"} - F["🔍 FEEDBACK
LogAnalysisBot reads the failure log
says what the notes were missing"] + E["🎯 EVAL
WritingBot solves each instance
using only the memory notes"] + C{"All resolved?"} + F["🔍 FEEDBACK
ReadingBot combines every
instance result into one message"] + T["🧠 TRAIN
ReadingBot re-reads the repo
and rewrites memory/"] E --> C C -- "yes" --> DONE(["✅ Done"]) C -- "no" --> F --> T --> E end - - CLI["cli.py
--model --task --max-rounds"] --> LOOP ``` -**Key idea:** the eval agent gets *no* extra hints — only the memory notes. -So a failing eval is direct evidence the notes are wrong or incomplete. +**Key idea:** the eval agent gets *no* hints beyond the memory notes. +A failing eval is therefore direct evidence the notes are wrong or incomplete. --- @@ -35,12 +35,12 @@ So a failing eval is direct evidence the notes are wrong or incomplete. | File | Role | One-liner | |---|---|---| -| `cli.py` | Entry point | Parses args, builds tasks, calls the orchestrator | -| `orchestrator.py` | Conductor | Owns the round loop, clones repo, wires train ↔ eval | -| `evalTask.py` | Contract | Abstract `EvalTask`: `run`, `check`, `build_feedback`, … | -| `task_registry.py` | Plugin table | `@register_task("name")` + auto-import of `eval/*` | -| `eval/swebenchverified.py` | A real task | One SWE-bench-Verified issue, graded by the official harness | -| `training/runner.py` | Trainer | One `ReadingBot` pass + `MemoryTool` | +| `cli.py` | Entry point | Resolves the workdir + config, builds the task, calls the orchestrator | +| `orchestrator.py` | Conductor | Clones the training repo, owns the round loop, wires eval ↔ training | +| `evalTask.py` | Contract | `EvalTask`: `parse_config`, `repo_url`, `eval` → `EvalOutcome` | +| `task_registry.py` | Plugin table | `@register_task("name")` + auto-import of everything in `eval/` | +| `eval/swebenchverified.py` | The task | A **set** of SWE-bench-Verified instances, graded by the official harness | +| `training/runner.py` | Trainer | One `ReadingBot` pass with a `MemoryTool` | | `training/training_instructions.md` | Trainer's brief | "Learn the repo, write notes, never edit code" | | `workdir.py` | Filing clerk | Every path under `workdir/` lives here — nothing is hard-coded elsewhere | @@ -53,64 +53,69 @@ sequenceDiagram autonumber participant O as orchestrator participant W as workdir - participant Task as EvalTask - participant Bot as WritingBot - participant Train as run_training_loop - - O->>W: load_round_memory(round N) - Note over W: copy memory/ ➜ rounds_/round_N/memory - O->>Task: run(eval_repo, memory_dir, model, log) - Task->>Task: setup() – clone/reset repo @ base commit - Task->>Bot: build_prompt() + MemoryTool(memory_dir) - Bot-->>Task: patch in repo + output - Task->>Task: check() – grade it (SWE-bench harness) - Task-->>O: EvalOutcome(passed, output, result) - - alt passed + participant T as SweBenchVerified + participant One as per-instance task + participant Train as run_training + + O->>W: take_memory_snapshot(round N) + Note over W: copy memory/ ➜ round_N/starting_memory_snapshot + O->>T: eval(memory_dir, model, eval_dir) + + loop every configured instance + T->>One: eval(...) + One->>One: setup() – clone/reset repo @ base commit + One->>One: WritingBot + MemoryTool(memory_dir) + One-->>T: BotRunResult (patch left in the checkout) + T->>One: check() – git diff ➜ SWE-bench harness + One-->>T: resolved / not resolved (+ test_output.txt) + end + + T-->>O: EvalOutcome(passed, score, feedback) + + alt every instance resolved O-->>O: return LoopResult(passed=True) - else failed - O->>Task: build_feedback(outcome, log) - Task-->>O: "your notes were missing X" - O->>Train: run_training_loop(feedback, memory_dir) × iterations - Train-->>W: notes updated in place + else some failed + O->>Train: run_training(feedback, memory_dir) + Train-->>W: memory/ rewritten in place end - O->>W: write result.json + save_round_memory(round N) - Note over W: copy round memory ➜ back up to memory/ + O->>W: write result.json ``` +`score` is the **fraction of instances resolved**; `passed` is true only when it reaches `1.0`. +An eval that raises is caught, recorded as `score = -1`, and the loop moves on — one bad +round never discards the rounds before it. + --- -## 4. Memory Lifecycle (the heart of it) +## 4. Memory Lifecycle -Memory is a **directory of markdown notes** that is copied down into each round, -mutated by the bots, then copied back up. +Memory is a **directory of markdown notes** with a single home. It is mutated in place; +each round snapshots its starting state so nothing is lost. ```mermaid flowchart TD - SEED["workdir/memory_seed/
immutable baseline snapshot"] - TOP["workdir/memory/
current best notes"] - R1["round_1/memory"] - R2["round_2/memory"] - R3["round_N/memory"] - - TOP -. "snapshot once, at run start" .-> SEED - TOP -->|load_round_memory| R1 - R1 -->|save_round_memory| TOP - TOP -->|load_round_memory| R2 - R2 -->|save_round_memory| TOP - TOP -->|load_round_memory| R3 - R3 -->|save_round_memory| TOP + TOP["workdir/memory/
the notes being optimized"] + S1["round_1/starting_memory_snapshot"] + S2["round_2/starting_memory_snapshot"] + S3["round_N/starting_memory_snapshot"] + + TOP -->|"snapshot at round start"| S1 + S1 -.->|"eval reads, training rewrites"| TOP + TOP -->|"snapshot at round start"| S2 + S2 -.->|"eval reads, training rewrites"| TOP + TOP -->|"snapshot at round start"| S3 + S3 -.->|"eval reads, training rewrites"| TOP ``` Rules that matter: -- **Replace, never merge.** `load`/`save` do `rmtree` + `copytree`, so deleted notes stay deleted - and stale files from a crashed round can't leak in. -- **`memory_seed` is written once.** It preserves the pre-run state, because `memory/` is - mutated in place all run long. -- **Memory carries across tasks.** Multiple task instances in one workdir share `memory/`, - so later instances inherit what earlier ones learned. +- **One live copy.** Both the eval agent and the training agent point at `workdir/memory`. +- **Snapshots are read-only history.** `round_N/starting_memory_snapshot` is what round N began with, + so you can diff what a round actually learned. +- **Re-running archives, it doesn't append.** `require_workdir` moves the old workdir to + `workdir_backup_` and carries `task_config.yaml`, `memory/` and `repo/` forward, + so a new run resumes from prior knowledge with clean round output. --- @@ -118,69 +123,62 @@ Rules that matter: ```text workdir/ -├── config.yaml # repo URL + task_args -├── repo/ # persistent clone — TRAINING only -├── eval_repo/ # task-managed clone — EVAL only (reset each round) -├── memory_seed/ # baseline snapshot (write-once) -├── memory/ # current best notes ← the thing being optimized -└── rounds_/ # per-task-instance, so instances never collide - └── round_N/ - ├── memory/ # this round's working copy of the notes - └── eval/ - ├── eval.log # agent output + harness logs (feeds LogAnalysisBot) - └── result.json +├── task_config.yaml # instance_id_list: [...] or repo: django/django +├── repo/ # training checkout, reused across rounds +├── memory/ # the notes being optimized ← the thing under test +└── rounds/round_N/ + ├── starting_memory_snapshot/ # memory/ as it looked when the round began + ├── logs/ # training logs + └── eval/ # owned entirely by the eval task + ├── eval_repo/ # shared checkout, reset per instance + ├── logs/_log.txt + └── result.json ``` -Two repos on purpose: the eval task wipes/resets its checkout every round, which -would otherwise destroy the training checkout. +Two checkouts on purpose: the eval task resets `eval_repo/` for every instance, which would +otherwise destroy the training checkout in `repo/`. --- -## 6. Two Modes - -```mermaid -flowchart LR - A["orchestrator.run(task=?)"] - A -->|"task is None"| B["Train-only
N training passes, empty feedback
round 1 is just a scratch dir"] - A -->|"task given"| C["run_train_eval_loop
up to max_rounds"] -``` +## 6. Running It ```bash -# train only -python -m microbots.auto_memory.cli --model azure-openai/gpt-5.5 +# task_config.yaml selects the eval set, by ID list... +# instance_id_list: +# - django__django-11099 +# ...or by repo: +# repo: django/django -# train + eval against a SWE-bench instance python -m microbots.auto_memory.cli \ --model azure-openai/gpt-5.5 \ --task swebenchverified \ - --max-rounds 5 --training-iterations 10 + --workdir ./workdir \ + --max-rounds 5 ``` +All instances in one config must belong to the **same repo** — that repo is what the training +agent learns, and `SweBenchVerified.repo_url()` derives it from the dataset. + --- ## 7. Adding a New Eval Task -Drop a module in `eval/` — `discover_tasks()` imports everything in that package, -so the `@register_task` decorator fires and the name shows up in `--task`. No central -factory to edit. +Drop a module in `eval/`. `discover_tasks()` imports everything in that package, so the +`@register_task` decorator fires and the name appears in `--task`. No central factory to edit. ```python @register_task("mytask") class MyTask(EvalTask): - @classmethod - def from_config(cls, task_args: dict) -> list["EvalTask"]: - ... # one instance per unit of work - - def run(self, repo_path, memory_dir, model, log_path) -> EvalOutcome: - ... # you drive setup/build_prompt/check yourself + def parse_config(self, config_file: Path) -> None: + ... # load your settings; set self._repo_url or override repo_url() - def build_feedback(self, outcome, repo_path, model, log_path) -> str: - ... # turn the failure log into "what the notes should say" + def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: + ... # run every unit of work, return one combined outcome ``` -Required: `from_config`, `run`, `build_feedback`. -Optional hooks (`setup`, `build_prompt`, `check`, `teardown`, `build_result`, `task_id`) -are **not** called automatically — your `run` decides. +Required: `eval`. `parse_config` and `repo_url` have working defaults driven by the config +file's `repo` key. The base `__init__` calls `parse_config` for you — so if you override +`__init__`, initialize your own state *before* calling it. --- @@ -188,9 +186,11 @@ are **not** called automatically — your `run` decides. | Where it breaks | What happens | |---|---| -| Agent run raises | Caught in `run`; logged; round fails with the exception as the reason | -| `build_feedback` / retraining raises | Logged; loop **continues to the next round** without retraining | -| Repo dir exists with wrong `origin` | Removed and re-cloned (never silently trains on wrong code) | +| Agent run raises | Caught per instance; logged; that instance counts as failed | +| Harness never writes `test_output.txt` | `error` falls back to the harness's console output | +| Feedback bot unavailable | Falls back to the raw concatenated per-instance results | +| `task.eval` raises | Logged; recorded as `score = -1`; loop continues to the next round | +| Training raises | Logged; loop continues to the next round without retraining | | `max_rounds` exhausted | `LoopResult(passed=False)` with every round's outcome | -Whatever happens, the `finally` block still writes `result.json` and saves the round's memory. +Whatever happens, the round's `result.json` is still written. diff --git a/src/microbots/auto_memory/cli.py b/src/microbots/auto_memory/cli.py index f4ba0da..e3f2294 100644 --- a/src/microbots/auto_memory/cli.py +++ b/src/microbots/auto_memory/cli.py @@ -14,7 +14,7 @@ from microbots.auto_memory.orchestrator import run from microbots.auto_memory.task_registry import TASK_REGISTRY, discover_tasks -from microbots.auto_memory.workdir import require_workdir, resolve_workdir +from microbots.auto_memory.workdir import config_path, require_workdir, resolve_workdir logger = logging.getLogger(__name__) @@ -78,7 +78,7 @@ def main(argv: list[str] | None = None) -> None: require_workdir(workdir) if not args.config_file: - config_file = workdir / "task_config.yaml" + config_file = config_path(workdir) else: config_file = args.config_file if not config_file.is_file(): diff --git a/src/microbots/auto_memory/eval/__init__.py b/src/microbots/auto_memory/eval/__init__.py new file mode 100644 index 0000000..f49433a --- /dev/null +++ b/src/microbots/auto_memory/eval/__init__.py @@ -0,0 +1 @@ +"""Eval tasks discovered and registered for the train <-> eval loop.""" diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index a840170..0507ef7 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -157,12 +157,12 @@ def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIF ) raise ValueError(f"instance_id not found: {instance_id}") -class SweBenchVerifiedTask_one(): - """SWE-bench-verified based evaluation task. +class SweBenchVerifiedTask_one: + """Runs and grades a single SWE-bench-verified instance. Checks out the instance's repo at its base commit, gives the agent - the issue's problem statement, and verifies the agent's patch using - the official SWE-bench evaluation harness. + the issue's problem statement, and verifies the resulting patch + with the official SWE-bench evaluation harness. Parameters ---------- @@ -170,29 +170,9 @@ class SweBenchVerifiedTask_one(): The dataset instance this task evaluates against. """ - def __init__(self, instance: SweBenchInstance | None = None): - """Initialize the task, optionally for a single dataset instance. - - Parameters - ---------- - instance : SweBenchInstance | None - The dataset instance this task evaluates against. May be - omitted and set later via ``self.instance``, but must be - set before any other method on this task is called. - """ + def __init__(self, instance: SweBenchInstance): self.instance = instance - @property - def task_id(self) -> str: - """Return this instance's SWE-bench-verified ``instance_id``. - - Returns - ------- - str - The dataset instance's ``instance_id``. - """ - return self.instance.instance_id - def setup(self, repo_path: str) -> None: """Clone the instance's repo, or reset it, to its base commit. @@ -344,26 +324,28 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> BotRunResul ) def eval(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> BotRunResult: - """Run one eval iteration: setup -> build_prompt -> WritingBot -> check. + """Check out the repo and let the agent attempt the issue. + + Grading is deliberately left to ``check``, which the caller runs + afterwards against the same checkout. Parameters ---------- repo_path : str - Absolute path to the repo to run the eval round against. + Absolute path to check the instance's repo out into. memory_dir : str Directory containing memory files to give the agent via ``MemoryTool``. model : str The model to use, in the format ``/``. log_path : str - Path to write this round's log to. Caller-provided, so the - log persists under the run's own layout. + Path to write this instance's log to. Truncated on entry. Returns ------- BotRunResult - The result of this eval round, including the agent's output, - the check verdict. + The agent's run result, or a failed result carrying the + exception if the attempt raised. """ Path(log_path).parent.mkdir(parents=True, exist_ok=True) Path(log_path).write_text("") @@ -397,34 +379,46 @@ def eval(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Bo @register_task("swebenchverified") class SweBenchVerified(EvalTask): - """SWE-bench-verified based evaluation task. + """Evaluates memory against a set of SWE-bench-verified instances. - It takes the memory provided by the training agent and runs - all the selected SWE-bench-verified instances. Then provides - a combined score and feedback. + Every instance in the configured set is attempted with the same + memory, and the round's score is the fraction that the harness + marks resolved. """ def __init__(self, config_file: Path) -> None: - # No need to call the base-class init + # dataset must exist before parse_config populates it. self.dataset: list[SweBenchInstance] = [] self.parse_config(config_file=config_file) def repo_url(self) -> str: - """Return the URL of the repo for the training agent. + """Return the clone URL of the repo the instances belong to. - Returns: - str: The URL of the repo for the training agent. + Returns + ------- + str + The training repo's clone URL. ``parse_config`` guarantees + every instance shares one repo. """ return f"https://github.com/{self.dataset[0].repo}.git" def parse_config(self, config_file: Path) -> None: - """Parse the configuration file for the task. - The config file is a yaml file. It will have array of "instance_id" - or "repo" as the root object. Gather it and load the dataset to - the object variable dataset. + """Load the instances this task evaluates from ``config_file``. + + The YAML file selects instances either by an + ``instance_id_list`` of dataset IDs, or by a ``repo`` naming a + SWE-bench repo such as ``django/django``. - Args: - config_file (Path): Path to the configuration file. + Parameters + ---------- + config_file : Path + Path to the task's YAML config file. + + Raises + ------ + ValueError + If the selected instances span more than one repo, or if + the config selects no instances at all. """ with open(config_file, "r") as f: @@ -453,16 +447,23 @@ def parse_config(self, config_file: Path) -> None: raise ValueError("No instances loaded for evaluation.") def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: - """Runs the evaluation agent with the memory on all the eval instances - and produces a cumulative feedback. + """Attempt every configured instance and combine the results. - Args: - memory_dir (str): Path to the directory containing the agent's memory. - model (str): The model identifier used for evaluation. - log_path (str): Path to the log file for recording evaluation details. + Parameters + ---------- + memory_dir : str + Directory containing the memory notes to evaluate. + model : str + The model to use, in the format ``/``. + eval_dir : str + Directory this round's eval owns; holds the shared checkout + and one log file per instance. - Returns: - EvalOutcome: The outcome of the evaluation, including whether it passed, the output, and the result. + Returns + ------- + EvalOutcome + ``score`` is the fraction of instances resolved, and + ``passed`` is true only when every one of them was. """ eval_path = Path(eval_dir) eval_repo_path = eval_path / "eval_repo" @@ -503,15 +504,22 @@ def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: ) def _combine_result_feedback(self, results: list[BotRunResult], model: str, eval_repo: str) -> str: - """ - Combines the feedback from multiple BotRunResult instances into a single feedback string. - Args: - results (list[BotRunResult]): List of individual bot run results. - model (str): The model identifier used for evaluation. - eval_repo (str): Path to the evaluation repository. - - Returns: - str: Combined feedback from all results. + """Summarize every instance's result into one feedback string. + + Parameters + ---------- + results : list[BotRunResult] + One result per attempted instance. + model : str + The model to use, in the format ``/``. + eval_repo : str + Path to the evaluation checkout, mounted for the bot. + + Returns + ------- + str + The bot's summary, falling back to the raw concatenated + results if the bot is unavailable or fails. """ serialized_str = f"Total {len(results)} tests ran and their result and feedback:\n" diff --git a/src/microbots/auto_memory/evalTask.py b/src/microbots/auto_memory/evalTask.py index dd9762b..50a791c 100644 --- a/src/microbots/auto_memory/evalTask.py +++ b/src/microbots/auto_memory/evalTask.py @@ -1,8 +1,7 @@ """Defines the abstract eval task interface for the train <-> eval loop. -An ``EvalTask`` describes one unit of work: how to prepare a repo, what -prompt to give the agent, how to verify the agent's output, and how to -clean up afterward. +An ``EvalTask`` owns its own config, names the repo the training agent +should learn from, and runs one complete evaluation per round. """ from abc import ABC, abstractmethod @@ -14,16 +13,17 @@ @dataclass class EvalOutcome: - """Result of verifying whether an eval task was completed correctly. + """Result of one round's evaluation. Attributes ---------- passed : bool - Whether the agent's output satisfies the task's check. + Whether every unit of work in the round passed. score : float - A numeric score representing the quality of the agent's output. + Fraction of units that passed, or ``-1`` if the round errored. feedback : str - A short human-readable explanation of the pass/fail verdict. + Text describing what went wrong, fed to the next round's + training pass. """ passed: bool @@ -31,30 +31,46 @@ class EvalOutcome: feedback: str class EvalTask(ABC): - """Base class for a single evaluation task in the train <-> eval loop. + """Base class for an evaluation task in the train <-> eval loop. - Subclasses must implement ``run``. ``parse_config``, ``setup``, - ``check``, and ``teardown`` are optional hooks - subclasses may use to structure their own ``run`` implementation - (see ``SweBenchVerifiedTask`` for an example), but nothing in this - base class calls them automatically. + Subclasses must implement ``eval``. ``parse_config`` and + ``repo_url`` have working defaults driven by the config file's + ``repo`` key, and may be overridden by tasks that derive the repo + some other way (see ``SweBenchVerified``). """ + _repo_url: str | None = None + def __init__(self, config_file: Path) -> None: + # NOTE: Don't call this from child class unless you need to reuse + # the parse_config logic from here. super().__init__() self.parse_config(config_file=config_file) def repo_url(self) -> str: - """Return the URL of the repo for the training agent.""" + """Return the URL of the repo the training agent should learn from. + + Returns + ------- + str + The training repo's clone URL. + + Raises + ------ + ValueError + If the config had no ``repo`` key and the subclass did not + override this method. + """ if not self._repo_url: raise ValueError("Repo URL is not set in the config file." " Or you didn't override the base method.") return self._repo_url def parse_config(self, config_file: Path) -> None: - """Parse the task-specific config file. Importantly it - parses the config file and get the repo for the training - agent. + """Read the task's config file, recording the training repo URL. + + Called from ``__init__``. Subclasses that need more than the + ``repo`` key override this to load their own settings too. Parameters ---------- @@ -67,7 +83,7 @@ def parse_config(self, config_file: Path) -> None: @abstractmethod def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: - """Required. Run one eval iteration and return its outcome. + """Required. Run one full evaluation and return its outcome. Parameters ---------- @@ -77,12 +93,12 @@ def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: model : str The model to use, in the format ``/``. eval_dir: str - Path to run this round's eval. This directory is managed by - the eval task itself. It can have its cloned repo, logs, etc. + Directory this round's eval owns. The task decides what + goes in it (cloned repo, logs, and so on). Returns ------- EvalOutcome - The result of this eval round, including the agent's output, - the check verdict. + Whether the round passed, its score, and the feedback to + retrain on. """ diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index 78d7754..bf1a70e 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -106,35 +106,27 @@ def run_train_eval_loop( ) -> LoopResult: """Run an eval task in a loop, retraining on failure until it passes. - Each round loads the current top-level memory into its own - ``rounds_/round_N/memory`` (carried forward from the - previous round, or empty on round 1), then runs ``task.run(...)`` - against it, writing its log to a workdir-managed path - (``rounds_/round_N/eval/eval.log``) so it persists. Since - each eval task instance gets its own ``rounds_`` dir, - different instances sharing the same ``workdir`` never collide on - round numbers, and each instance's per-round memory is preserved - individually. If the task passes, the loop returns immediately. If - it fails, feedback is built from the round's log and used to - retrain via ``run_training`` (called ``training_iterations`` times, - each pass reusing the same round memory dir) before the next round. - Either way, the round's result is written to ``result.json`` and - its memory is saved back to the top-level memory dir before the - next round starts. + Memory lives in one place (``workdir/memory``) and is mutated in + place: each round snapshots it to + ``rounds/round_N/starting_memory_snapshot`` before the eval agent + reads it, so what the round began with stays recoverable. The task + then evaluates against that memory in its own + ``rounds/round_N/eval`` directory. Passing returns immediately; + failing feeds ``outcome.feedback`` to ``run_training``, which + rewrites memory for the next round. Either way the round's outcome + is written to ``result.json``. + + An eval that raises is logged and recorded as a failed outcome so + one bad round cannot discard the rounds before it. Parameters ---------- training_repo_path : str - Absolute path to the persistent repo checkout used only for - retraining (``run_training_loop``). Kept separate from - ``eval_repo_path`` since the task manages the latter's - lifecycle itself (clone/teardown each round). - training_repo_path : str - Absolute path to the persistent repo checkout used only for - retraining (``run_training_loop``). + Absolute path to the persistent checkout the training agent + reads. Separate from the eval checkout, which the task clones + and resets itself every round. workdir : Path - This run's workdir, used to carry memory forward between rounds - (see ``microbots.auto_memory.workdir``). + This run's workdir (see ``microbots.auto_memory.workdir``). model : str The model to use, in the format ``/``. task : EvalTask @@ -230,36 +222,25 @@ def run( task: EvalTask, max_rounds: int = 5, ) -> LoopResult: - """Run full train/eval loop, depending on ``task``. + """Clone the task's training repo, then run the full train/eval loop. Parameters ---------- workdir : Path This run's workdir (see ``microbots.auto_memory.workdir``), - holding ``task_config.yaml``, the shared repo clone, and all output. + holding the training clone, memory, and all round output. model : str The model to use, in the format ``/``. task : EvalTask - The eval task to run each round. + The eval task to run each round. It also supplies the training + repo's clone URL via ``repo_url()``. max_rounds : int Maximum number of train/eval rounds to attempt. Defaults to 5. - training_iterations : int - Number of training passes to run per retraining round, each - reusing the same round memory dir. Defaults to 10. Returns ------- LoopResult The eval loop's result. - - Raises - ------ - ValueError - If ``config`` has no ``repo`` entry. Every run needs a training - checkout (``run_training_loop`` always mounts - ``training_repo_path``, regardless of ``task``), so ``repo`` - must be configured even for tasks like ``SweBenchVerifiedTask`` - that manage their own separate eval checkout. """ training_repo_dir = repo_dir(workdir) clone_repo(task.repo_url(), training_repo_dir) diff --git a/src/microbots/auto_memory/task_registry.py b/src/microbots/auto_memory/task_registry.py index f8720ae..46a6bed 100644 --- a/src/microbots/auto_memory/task_registry.py +++ b/src/microbots/auto_memory/task_registry.py @@ -1,8 +1,9 @@ -"""Registry for constructing ``EvalTask`` instances by name. +"""Registry for looking up ``EvalTask`` classes by name. Tasks self-register via the ``@register_task`` decorator, so new task types can be added without editing a central if/elif factory function. -Callers (e.g. a CLI) look tasks up by name via ``create_task``. +Callers (e.g. a CLI) look a class up in ``TASK_REGISTRY`` and construct +it with the run's config file. """ import importlib @@ -22,8 +23,8 @@ def register_task(name: str) -> Callable[[type[EvalTask]], type[EvalTask]]: Parameters ---------- name : str - The key other code will use to look up this task via - ``create_task``, e.g. ``"swebenchverified"``. + The key other code will use to look this task up in + ``TASK_REGISTRY``, e.g. ``"swebenchverified"``. Returns ------- diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py index 2f7693c..3d35d63 100644 --- a/src/microbots/auto_memory/workdir.py +++ b/src/microbots/auto_memory/workdir.py @@ -5,7 +5,6 @@ outputs), so callers never hard-code layout details themselves. """ -import os import shutil import time from pathlib import Path @@ -30,17 +29,14 @@ """ Expected workdir structure: - workdir/ - | workdir/ ├── task_config.yaml - ├── repo/ - ├── memory/ - ├── rounds/round_n/ - │ └── logs/ <-- Contains only training log. Eval logs can be found inside the eval task - │ └── eval/ <-- Managed by the eval task - │ └── starting_memory_snapshot/ - └── logs/ + ├── repo/ <-- Training checkout, reused across rounds + ├── memory/ <-- Mutated in place; the run's living memory + └── rounds/round_n/ + ├── logs/ <-- Training logs; eval logs live under eval/ + ├── eval/ <-- Managed by the eval task + └── starting_memory_snapshot/ <-- memory/ as it looked when the round began """ @@ -63,40 +59,31 @@ def resolve_workdir(base: Path | None = None) -> Path: def require_workdir(workdir: Path) -> None: - """Validate that ``workdir`` exist. Create if not exist + """Prepare ``workdir`` for a fresh run, archiving any previous one. + + A run must start from clean round output, so an existing workdir is + archived to ``_backup_`` and recreated. The config, + memory, and training checkout are carried over from the archive so + the next run resumes from what the last one learned instead of + re-cloning and re-learning from scratch. Parameters ---------- workdir : Path - The workdir to validate. - + The workdir to prepare. Created if it does not exist. """ - # create workdir if not existing - if not workdir.exists(): - mem_dir = workdir / MEMORY_DIRNAME - mem_dir.mkdir(parents=True) - return - - workdir_parent = workdir.parent - workdir_backup = workdir_parent / f"{workdir.name}_backup_{int(time.time())}" - shutil.move(str(workdir), str(workdir_backup)) - workdir.mkdir(parents=True) - - task_config = workdir_backup / CONFIG_FILENAME - if task_config.exists(): - shutil.copy(task_config, workdir / CONFIG_FILENAME) - else: - raise FileNotFoundError(f"Task config file does not exist: {task_config}") - - mem_dir = workdir_backup / MEMORY_DIRNAME - if mem_dir.exists(): - shutil.copytree(mem_dir, workdir / MEMORY_DIRNAME) - else: - mem_dir.mkdir(parents=True) - - repo_dir = workdir_backup / REPO_DIRNAME - if repo_dir.exists(): - shutil.copytree(repo_dir, workdir / REPO_DIRNAME) + if workdir.exists(): + backup = workdir.parent / f"{workdir.name}_backup_{int(time.time())}" + shutil.move(str(workdir), str(backup)) + workdir.mkdir(parents=True) + # Moved rather than copied; repo/ can be hundreds of MB. + for name in (CONFIG_FILENAME, MEMORY_DIRNAME, REPO_DIRNAME): + carried_over = backup / name + if carried_over.exists(): + shutil.move(str(carried_over), str(workdir / name)) + + # Every round snapshots this, so it must exist even when empty. + (workdir / MEMORY_DIRNAME).mkdir(parents=True, exist_ok=True) def config_path(workdir: Path) -> Path: @@ -110,19 +97,16 @@ def config_path(workdir: Path) -> Path: Returns ------- Path - ``workdir/config.yaml``. + ``workdir/task_config.yaml``. """ return workdir / CONFIG_FILENAME def repo_dir(workdir: Path) -> Path: - """Return the path to the single cloned repo shared across rounds. + """Return the path to the training checkout shared across rounds. - Used only for training (both train-only mode and the eval loop's - retrain step): a persistent checkout that stays in place across - rounds. Eval tasks that manage their own repo checkout (e.g. - ``SweBenchVerifiedTask``, which clones a different repo/commit per - dataset instance) use ``eval_repo_dir`` instead, so the two never + Used only by the retraining step. Eval tasks clone and manage their + own checkout under the round's eval directory, so the two never collide. Parameters @@ -144,7 +128,7 @@ def memory_dir(workdir: Path) -> Path: Parameters ---------- workdir : Path - The run's workdir. + The run's workdir. Returns ------- @@ -155,22 +139,29 @@ def memory_dir(workdir: Path) -> Path: def take_memory_snapshot(mem_dir: Path, round_idx: int) -> None: - """Take a snapshot of the current memory directory for a specific round. + """Snapshot the memory dir as this round's starting point. + + Memory is mutated in place across rounds, so this preserves what + the round started from before training rewrites it. Parameters ---------- mem_dir : Path - The path to the current top-level memory directory. + The run's top-level memory directory. round_idx : int The 1-based round number. + + Raises + ------ + FileNotFoundError + If ``mem_dir`` does not exist. It may be empty, but the run's + layout must already have created it. """ if not mem_dir.exists(): - # At this point, mem_dir can be empty but it should exist raise FileNotFoundError(f"Memory directory does not exist: {mem_dir}") - workdir = Path(mem_dir).parent - snapshot_dir = round_dir(workdir, round_idx) / STARTING_MEMORY_SNAPSHOT_DIR - if os.path.exists(snapshot_dir): + snapshot_dir = round_dir(mem_dir.parent, round_idx) / STARTING_MEMORY_SNAPSHOT_DIR + if snapshot_dir.exists(): shutil.rmtree(snapshot_dir) shutil.copytree(mem_dir, snapshot_dir) @@ -197,7 +188,7 @@ def round_dir( def round_log_dir(workdir: Path, round_num: int) -> Path: - """Return the path to a round's training log. + """Return the path to a round's training log directory. Parameters ---------- @@ -209,8 +200,7 @@ def round_log_dir(workdir: Path, round_num: int) -> Path: Returns ------- Path - This round's log directory. Both training and eval logs - can be found inside with appropriate file names + ``workdir/rounds/round_{round_num}/logs``. """ return round_dir(workdir, round_num) / ROUND_LOG_DIR diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py index 7caffea..71e565d 100644 --- a/test/auto_memory/eval/test_swebenchverified.py +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -1,675 +1,214 @@ -"""Unit tests for microbots.auto_memory.eval.swebenchverified.""" +"""Unit tests for microbots.auto_memory.eval.swebenchverified. + +The SWE-bench dataset, git, the evaluation harness and every bot are +mocked, so these run without network access, Docker or an LLM. +""" import json -import os -import sys +import subprocess from pathlib import Path from unittest.mock import MagicMock, patch import pytest - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/"))) +import yaml from microbots.auto_memory.eval.swebenchverified import ( - SWE_BENCH_VERIFIED, SweBenchInstance, - SweBenchVerifiedTask, - _load_dataset_rows, - load_instance_using_id, - load_instances_of_repo, + SweBenchVerified, + SweBenchVerifiedTask_one, ) -from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome from microbots.MicroBot import BotRunResult MODULE = "microbots.auto_memory.eval.swebenchverified" +INSTANCE = SweBenchInstance( + instance_id="django__django-11099", + repo="django/django", + base_commit="abc123", + problem_statement="UsernameValidator allows trailing newline", +) -@pytest.fixture(autouse=True) -def _clear_dataset_cache(): - """Clear ``_load_dataset_rows``'s cache so each test's ``load_dataset`` mock takes effect.""" - _load_dataset_rows.cache_clear() - yield - _load_dataset_rows.cache_clear() - - -def _fake_rows(): - return [ - { - "instance_id": "django__django-1", - "repo": "django/django", - "base_commit": "abc123", - "problem_statement": "fix bug 1", - }, - { - "instance_id": "astropy__astropy-1", - "repo": "astropy/astropy", - "base_commit": "def456", - "problem_statement": "fix bug 2", - }, - { - "instance_id": "django__django-2", - "repo": "django/django", - "base_commit": "ghi789", - "problem_statement": "fix bug 3", - }, - ] - - -# --------------------------------------------------------------------------- -# load_instances_of_repo / load_instance_using_id -# --------------------------------------------------------------------------- - -@pytest.mark.unit -@patch("datasets.load_dataset") -def test_load_instances_of_repo_filters_by_repo(mock_load_dataset): - mock_load_dataset.return_value = _fake_rows() - - instances = load_instances_of_repo(repo="django/django") - - assert [i.instance_id for i in instances] == ["django__django-1", "django__django-2"] - assert all(isinstance(i, SweBenchInstance) for i in instances) - - -@pytest.mark.unit -@patch("datasets.load_dataset") -def test_load_instances_of_repo_returns_all_when_repo_none(mock_load_dataset): - mock_load_dataset.return_value = _fake_rows() - - instances = load_instances_of_repo(repo=None) - - assert len(instances) == 3 - - -@pytest.mark.unit -@patch("datasets.load_dataset") -def test_load_instance_using_id_returns_matching_instance(mock_load_dataset): - mock_load_dataset.return_value = _fake_rows() - - instance = load_instance_using_id("astropy__astropy-1") - - assert instance.repo == "astropy/astropy" - assert instance.problem_statement == "fix bug 2" - - -@pytest.mark.unit -@patch("datasets.load_dataset") -def test_load_instance_using_id_raises_when_not_found(mock_load_dataset): - mock_load_dataset.return_value = _fake_rows() - - with pytest.raises(ValueError, match="not found"): - load_instance_using_id("does-not-exist") - - -@pytest.mark.unit -@patch("datasets.load_dataset") -def test_dataset_rows_are_cached_across_repeated_calls(mock_load_dataset): - """``load_dataset`` should only be called once per ``dataset_name``, even - across multiple ``load_instances_of_repo``/``load_instance_using_id`` calls.""" - mock_load_dataset.return_value = _fake_rows() - - load_instances_of_repo(repo="django/django") - load_instances_of_repo(repo=None) - load_instance_using_id("astropy__astropy-1") - - mock_load_dataset.assert_called_once() - - -@pytest.mark.unit -def test_load_dataset_rows_raises_helpful_error_when_datasets_not_installed(): - with patch.dict(sys.modules, {"datasets": None}): - with pytest.raises(ImportError, match=r"pip install 'microbots\[training\]'"): - _load_dataset_rows(SWE_BENCH_VERIFIED) - - -# --------------------------------------------------------------------------- -# SweBenchVerifiedTask.setup / build_prompt -# --------------------------------------------------------------------------- - -def _instance(): - return SweBenchInstance( - instance_id="django__django-1", - repo="django/django", - base_commit="abc123", - problem_statement="fix the bug", - ) - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_setup_clones_and_checks_out_base_commit_when_repo_missing(mock_run, tmp_path): - repo_path = tmp_path / "repo" - task = SweBenchVerifiedTask(_instance()) - task.setup(str(repo_path)) - - clone_call, checkout_call = mock_run.call_args_list - assert clone_call.args[0] == [ - "git", "clone", "https://github.com/django/django.git", str(repo_path) - ] - assert checkout_call.args[0] == ["git", "checkout", "abc123"] - assert checkout_call.kwargs["cwd"] == str(repo_path) - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_setup_resets_instead_of_recloning_when_origin_matches(mock_run, tmp_path): - repo_path = tmp_path / "repo" - repo_path.mkdir() - mock_run.return_value = MagicMock( - returncode=0, stdout="https://github.com/django/django.git\n" - ) - task = SweBenchVerifiedTask(_instance()) - task.setup(str(repo_path)) - - origin_call, reset_call, clean_call = mock_run.call_args_list - assert origin_call.args[0] == ["git", "remote", "get-url", "origin"] - assert origin_call.kwargs["cwd"] == str(repo_path) - assert reset_call.args[0] == ["git", "reset", "--hard", "abc123"] - assert reset_call.kwargs["cwd"] == str(repo_path) - assert clean_call.args[0] == ["git", "clean", "-fd"] - assert clean_call.kwargs["cwd"] == str(repo_path) - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_setup_removes_and_reclones_when_origin_mismatched(mock_run, tmp_path): - repo_path = tmp_path / "repo" - repo_path.mkdir() - (repo_path / "stale_file.txt").write_text("leftover from a different repo") - mock_run.return_value = MagicMock( - returncode=0, stdout="https://github.com/other/repo.git\n" - ) - task = SweBenchVerifiedTask(_instance()) - task.setup(str(repo_path)) - - assert not (repo_path / "stale_file.txt").exists() - origin_call, clone_call, checkout_call = mock_run.call_args_list - assert origin_call.args[0] == ["git", "remote", "get-url", "origin"] - assert clone_call.args[0] == [ - "git", "clone", "https://github.com/django/django.git", str(repo_path) - ] - assert checkout_call.args[0] == ["git", "checkout", "abc123"] - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_setup_removes_and_reclones_when_repo_path_not_a_git_repo(mock_run, tmp_path): - repo_path = tmp_path / "repo" - repo_path.mkdir() - mock_run.return_value = MagicMock(returncode=128, stdout="") - task = SweBenchVerifiedTask(_instance()) - task.setup(str(repo_path)) - - assert not repo_path.exists() - origin_call, clone_call, checkout_call = mock_run.call_args_list - assert origin_call.args[0] == ["git", "remote", "get-url", "origin"] - assert clone_call.args[0] == [ - "git", "clone", "https://github.com/django/django.git", str(repo_path) - ] - - -@pytest.mark.unit -def test_build_prompt_returns_problem_statement(): - task = SweBenchVerifiedTask(_instance()) - assert task.build_prompt() == "fix the bug" - - -@pytest.mark.unit -def test_task_id_is_instance_id(): - task = SweBenchVerifiedTask(_instance()) - assert task.task_id == "django__django-1" +def _config(tmp_path: Path, **body) -> Path: + path = tmp_path / "task_config.yaml" + path.write_text(yaml.safe_dump(body)) + return path -@pytest.mark.unit -def test_build_result_includes_dataset_fields(): - task = SweBenchVerifiedTask(_instance()) - outcome = EvalOutcome( - passed=True, - output="agent output", - result=CallbackResult(passed=True, reason="resolved"), - ) - assert task.build_result(outcome) == { - "passed": True, - "reason": "resolved", - "instance_id": "django__django-1", - "repo": "django/django", - "base_commit": "abc123", - } +def _task(tmp_path: Path, **body) -> SweBenchVerified: + with patch(f"{MODULE}.load_instance_using_id", return_value=INSTANCE): + return SweBenchVerified(_config(tmp_path, **body)) # --------------------------------------------------------------------------- -# SweBenchVerifiedTask.build_feedback +# parse_config / repo_url # --------------------------------------------------------------------------- -def _failed_outcome(reason: str = "tests failed", output: str = "agent output") -> EvalOutcome: - return EvalOutcome( - passed=False, - output=output, - result=CallbackResult(passed=False, reason=reason), - ) - - @pytest.mark.unit -@patch(f"{MODULE}.LogAnalysisBot") -def test_build_feedback_returns_bot_result_on_success(mock_bot_cls): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult( - status=True, result="root cause: missing edge case handling", error=None - ) - mock_bot_cls.return_value = mock_bot - - task = SweBenchVerifiedTask(_instance()) - outcome = _failed_outcome() - - feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log") +def test_parse_config_loads_the_listed_instances(tmp_path): + task = _task(tmp_path, instance_id_list=["django__django-11099"]) - assert feedback == "root cause: missing edge case handling" - mock_bot_cls.assert_called_once_with(model="azure-openai/gpt-4o", folder_to_mount="/repo") - mock_bot.run.assert_called_once() - assert mock_bot.run.call_args.kwargs["file_name"] == "/tmp/some.log" + assert [i.instance_id for i in task.dataset] == ["django__django-11099"] + assert task.repo_url() == "https://github.com/django/django.git" @pytest.mark.unit -@patch(f"{MODULE}.LogAnalysisBot") -def test_build_feedback_falls_back_when_bot_status_false(mock_bot_cls): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed") - mock_bot_cls.return_value = mock_bot - - task = SweBenchVerifiedTask(_instance()) - outcome = _failed_outcome(reason="tests failed", output="some output") - - feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log") +def test_parse_config_loads_every_instance_of_a_repo(tmp_path): + with patch(f"{MODULE}.load_instances_of_repo", return_value=[INSTANCE, INSTANCE]) as loader: + task = SweBenchVerified(_config(tmp_path, repo="django/django")) - assert "some output" in feedback - assert "tests failed" in feedback + loader.assert_called_once_with(repo="django/django") + assert len(task.dataset) == 2 @pytest.mark.unit -@patch(f"{MODULE}.LogAnalysisBot") -def test_build_feedback_falls_back_when_result_is_empty(mock_bot_cls): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="", error=None) - mock_bot_cls.return_value = mock_bot +def test_parse_config_rejects_instances_from_different_repos(tmp_path): + other = SweBenchInstance("flask__flask-1", "pallets/flask", "def456", "boom") - task = SweBenchVerifiedTask(_instance()) - outcome = _failed_outcome(reason="assertion error", output="agent tried X") - - feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log") - - assert "agent tried X" in feedback - assert "assertion error" in feedback + with patch(f"{MODULE}.load_instance_using_id", side_effect=[INSTANCE, other]): + with pytest.raises(ValueError, match="Conflicting repos"): + SweBenchVerified(_config(tmp_path, instance_id_list=["a", "b"])) @pytest.mark.unit -@patch(f"{MODULE}.LogAnalysisBot") -def test_build_feedback_falls_back_when_result_is_none(mock_bot_cls): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result=None, error=None) - mock_bot_cls.return_value = mock_bot - - task = SweBenchVerifiedTask(_instance()) - outcome = _failed_outcome() - - feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log") - - assert "Evaluation failed" in feedback +def test_parse_config_rejects_an_empty_selection(tmp_path): + with pytest.raises(ValueError, match="No instances loaded"): + SweBenchVerified(_config(tmp_path, note="nothing selected")) # --------------------------------------------------------------------------- -# SweBenchVerifiedTask.check +# check # --------------------------------------------------------------------------- -def _make_fake_subprocess_run(resolved: bool, raise_on_harness: bool = False): - """Build a subprocess.run stand-in that fakes git diff + the harness call.""" - - def _fake_run(cmd, **kwargs): - if cmd[:2] == ["git", "diff"]: - return MagicMock(stdout="diff --git a/x.py b/x.py\n+fix", stderr="", returncode=0) - if "swebench.harness.run_evaluation" in cmd: - if raise_on_harness: - raise RuntimeError("harness crashed") - run_id = cmd[cmd.index("--run_id") + 1] - report_dir = Path(kwargs["cwd"]) - instance_id = cmd[cmd.index("--instance_ids") + 1] - instance_log_dir = ( - report_dir / "logs" / "run_evaluation" / run_id - / "microbots-eval-agent" / instance_id - ) - instance_log_dir.mkdir(parents=True) - report = {instance_id: {"resolved": resolved}} - (instance_log_dir / "report.json").write_text(json.dumps(report)) - return MagicMock(stdout="harness ran\n", stderr="", returncode=0) - return MagicMock(stdout="", stderr="", returncode=0) - - return _fake_run - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_check_marks_untracked_files_intent_to_add_before_diffing(mock_run, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=True) - log_path = tmp_path / "check.log" - log_path.write_text("") - - task = SweBenchVerifiedTask(_instance()) - task.check("/repo", "agent output", str(log_path)) - - calls = [c.args[0] for c in mock_run.call_args_list] - add_idx = calls.index(["git", "add", "--intent-to-add", "."]) - diff_idx = calls.index(["git", "diff", "--binary"]) - assert add_idx < diff_idx - assert mock_run.call_args_list[add_idx].kwargs["cwd"] == "/repo" - +def _fake_harness(report: dict, test_output: str): + """Stand in for the SWE-bench harness, writing its usual artifacts.""" -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_check_passed_true_when_report_marks_resolved(mock_run, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=True) - log_path = tmp_path / "check.log" - log_path.write_text("") + def run(cmd, *args, **kwargs): + if cmd[:2] == ["git", "diff"] or cmd[:2] == ["git", "add"]: + return subprocess.CompletedProcess(cmd, 0, stdout="diff --git a b\n", stderr="") - task = SweBenchVerifiedTask(_instance()) - result = task.check("/repo", "agent output", str(log_path)) + report_dir = Path(kwargs["cwd"]) + run_id = cmd[cmd.index("--run_id") + 1] + log_dir = ( + report_dir / "logs" / "run_evaluation" / run_id + / "microbots-eval-agent" / INSTANCE.instance_id + ) + log_dir.mkdir(parents=True) + (log_dir / "report.json").write_text(json.dumps(report)) + (log_dir / "test_output.txt").write_text(test_output) + return subprocess.CompletedProcess(cmd, 0, stdout="harness done\n", stderr="") - assert result.passed is True - assert result.reason == "resolved" + return run @pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_check_passed_false_when_report_marks_not_resolved(mock_run, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=False) - log_path = tmp_path / "check.log" +def test_check_passes_when_the_harness_resolves_the_instance(tmp_path): + log_path = tmp_path / "instance.log" log_path.write_text("") + harness = _fake_harness({INSTANCE.instance_id: {"resolved": True}}, "OK") - task = SweBenchVerifiedTask(_instance()) - result = task.check("/repo", "agent output", str(log_path)) + with patch(f"{MODULE}.subprocess.run", side_effect=harness): + result = SweBenchVerifiedTask_one(INSTANCE).check("/repo", "", str(log_path)) - assert result.passed is False - assert result.reason == "not resolved" + assert result.status + assert result.result == "resolved" + assert result.error is None @pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_check_passed_false_when_report_file_never_written(mock_run, tmp_path): - # harness call succeeds but never writes a report file (e.g. it errored internally) - def _fake_run(cmd, **kwargs): - if cmd[:2] == ["git", "diff"]: - return MagicMock(stdout="diff", stderr="", returncode=0) - return MagicMock(stdout="", stderr="", returncode=1) - - mock_run.side_effect = _fake_run - log_path = tmp_path / "check.log" - log_path.write_text("") - - task = SweBenchVerifiedTask(_instance()) - result = task.check("/repo", "agent output", str(log_path)) - - assert result.passed is False - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_check_appends_to_log_file_without_truncating_existing_content(mock_run, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=True) - log_path = tmp_path / "check.log" - log_path.write_text("Agent output:\nprevious content\n") - - task = SweBenchVerifiedTask(_instance()) - task.check("/repo", "agent output", str(log_path)) - - content = log_path.read_text() - assert "previous content" in content - assert "harness ran" in content - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_check_appends_instance_log_and_test_output_when_present(mock_run, tmp_path): - def _fake_run(cmd, **kwargs): - if cmd[:2] == ["git", "diff"]: - return MagicMock(stdout="diff", stderr="", returncode=0) - if "swebench.harness.run_evaluation" in cmd: - run_id = cmd[cmd.index("--run_id") + 1] - report_dir = Path(kwargs["cwd"]) - instance_id = cmd[cmd.index("--instance_ids") + 1] - - instance_log_dir = ( - report_dir / "logs" / "run_evaluation" / run_id - / "microbots-eval-agent" / instance_id - ) - instance_log_dir.mkdir(parents=True) - report = {instance_id: {"resolved": True}} - (instance_log_dir / "report.json").write_text(json.dumps(report)) - (instance_log_dir / "run_instance.log").write_text("build+test steps") - (instance_log_dir / "test_output.txt").write_text("FAILED test_foo") - - return MagicMock(stdout="harness ran\n", stderr="", returncode=0) - return MagicMock(stdout="", stderr="", returncode=0) - - mock_run.side_effect = _fake_run - log_path = tmp_path / "check.log" - log_path.write_text("") - - task = SweBenchVerifiedTask(_instance()) - task.check("/repo", "agent output", str(log_path)) - - content = log_path.read_text() - assert "run_instance.log" in content - assert "build+test steps" in content - assert "test_output.txt" in content - assert "FAILED test_foo" in content - - -@pytest.mark.unit -@patch(f"{MODULE}.shutil.rmtree") -@patch(f"{MODULE}.subprocess.run") -def test_check_cleans_up_pred_path_and_report_dir_on_success(mock_run, mock_rmtree, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=True) - log_path = tmp_path / "check.log" - log_path.write_text("") - - task = SweBenchVerifiedTask(_instance()) - task.check("/repo", "agent output", str(log_path)) - - mock_rmtree.assert_called_once() - - -@pytest.mark.unit -@patch(f"{MODULE}.shutil.rmtree") -@patch(f"{MODULE}.subprocess.run") -def test_check_cleans_up_even_when_harness_raises(mock_run, mock_rmtree, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=True, raise_on_harness=True) - log_path = tmp_path / "check.log" +def test_check_reports_the_test_output_when_the_instance_is_unresolved(tmp_path): + log_path = tmp_path / "instance.log" log_path.write_text("") + harness = _fake_harness( + {INSTANCE.instance_id: {"resolved": False}}, + "FAILED tests/test_validators.py::test_trailing_newline", + ) - task = SweBenchVerifiedTask(_instance()) - with pytest.raises(RuntimeError, match="harness crashed"): - task.check("/repo", "agent output", str(log_path)) + with patch(f"{MODULE}.subprocess.run", side_effect=harness): + result = SweBenchVerifiedTask_one(INSTANCE).check("/repo", "", str(log_path)) - mock_rmtree.assert_called_once() + assert not result.status + assert "test_trailing_newline" in result.error + assert "test_trailing_newline" in log_path.read_text() # --------------------------------------------------------------------------- -# SweBenchVerifiedTask.run +# eval # --------------------------------------------------------------------------- @pytest.mark.unit -@patch(f"{MODULE}.MemoryTool") -@patch(f"{MODULE}.WritingBot") -def test_run_calls_setup_build_prompt_check_in_order(mock_bot_cls, mock_memory_tool, tmp_path): - from microbots.MicroBot import BotRunResult - - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="agent did stuff", error=None) - mock_bot_cls.return_value = mock_bot - - task = SweBenchVerifiedTask(_instance()) - calls = [] - task.setup = lambda repo_path: calls.append(("setup", repo_path)) - task.build_prompt = lambda: "do the task" - task.check = lambda repo_path, agent_output, log_path: ( - calls.append(("check", repo_path, agent_output, log_path)) - or CallbackResult(passed=True, reason="ok") - ) - - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) - - assert calls[0] == ("setup", "/repo") - assert calls[1] == ("check", "/repo", "agent did stuff", str(tmp_path / "eval.log")) - assert outcome.passed is True - assert outcome.output == "agent did stuff" - - -@pytest.mark.unit -@patch(f"{MODULE}.MemoryTool") -@patch(f"{MODULE}.WritingBot") -def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_tool, tmp_path): - from microbots.MicroBot import BotRunResult - - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) - mock_bot_cls.return_value = mock_bot - - task = SweBenchVerifiedTask(_instance()) - task.setup = lambda repo_path: None - task.build_prompt = lambda: "do the task" - seen_log_exists = {} - - def _check(repo_path, agent_output, log_path): - seen_log_exists["exists"] = os.path.exists(log_path) - return CallbackResult(passed=True, reason="ok") - - task.check = _check - - task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) - - assert seen_log_exists["exists"] is True - - -@pytest.mark.unit -@patch(f"{MODULE}.MemoryTool") -@patch(f"{MODULE}.WritingBot") -def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool, tmp_path): - from microbots.MicroBot import BotRunResult - - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed") - mock_bot_cls.return_value = mock_bot - - task = SweBenchVerifiedTask(_instance()) - check_calls = [] - task.setup = lambda repo_path: None - task.build_prompt = lambda: "do the task" - task.check = lambda *a: check_calls.append(a) - - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) - - assert check_calls == [] - assert outcome.passed is False - assert "bot crashed" in outcome.result.reason - - -@pytest.mark.unit -@patch(f"{MODULE}.MemoryTool") -@patch(f"{MODULE}.WritingBot") -def test_run_converts_build_prompt_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool, tmp_path): - task = SweBenchVerifiedTask(_instance()) - task.setup = lambda repo_path: None - - def _build_prompt(): - raise ValueError("bad prompt") - - task.build_prompt = _build_prompt - - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) +def test_eval_scores_the_fraction_of_resolved_instances(tmp_path): + task = _task(tmp_path, instance_id_list=["django__django-11099"]) + task.dataset = [INSTANCE, INSTANCE] - assert outcome.passed is False - assert "bad prompt" in outcome.result.reason - with open(str(tmp_path / "eval.log")) as f: - assert "bad prompt" in f.read() - - -@pytest.mark.unit -def test_run_converts_setup_exception_to_failed_outcome(tmp_path): - task = SweBenchVerifiedTask(_instance()) - - def _setup(repo_path): - raise RuntimeError("clone failed") - - task.setup = _setup - - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) - - assert outcome.passed is False - assert "clone failed" in outcome.result.reason - with open(str(tmp_path / "eval.log")) as f: - assert "clone failed" in f.read() - - -@pytest.mark.unit -@patch(f"{MODULE}.MemoryTool") -@patch(f"{MODULE}.WritingBot") -def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool, tmp_path): - from microbots.MicroBot import BotRunResult - - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) - mock_bot_cls.return_value = mock_bot - - task = SweBenchVerifiedTask(_instance()) - task.setup = lambda repo_path: None - task.build_prompt = lambda: "do the task" - - def _check(repo_path, agent_output, log_path): - raise RuntimeError("check exploded") - - task.check = _check - - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) + agent_ran = BotRunResult(status=True, result="patched", error=None) + verdicts = [ + BotRunResult(status=True, result="resolved", error=None), + BotRunResult(status=False, result="not resolved", error="tests failed"), + ] - assert outcome.passed is False - assert "check exploded" in outcome.result.reason + with patch.object(SweBenchVerifiedTask_one, "setup"), \ + patch.object(SweBenchVerifiedTask_one, "check", side_effect=verdicts), \ + patch(f"{MODULE}.WritingBot") as writing_bot, \ + patch(f"{MODULE}.MemoryTool"), \ + patch(f"{MODULE}.ReadingBot") as reading_bot: + writing_bot.return_value.run.return_value = agent_ran + reading_bot.return_value.run.return_value = BotRunResult( + status=True, result="one instance still fails", error=None + ) + outcome = task.eval(str(tmp_path / "memory"), "azure-openai/gpt-4o", str(tmp_path / "eval")) + assert outcome.score == 0.5 + assert not outcome.passed + assert outcome.feedback == "one instance still fails" -# --------------------------------------------------------------------------- -# SweBenchVerifiedTask.from_config -# --------------------------------------------------------------------------- @pytest.mark.unit -@patch(f"{MODULE}.load_instance_using_id") -def test_from_config_uses_instance_id_when_given(mock_load_instance_using_id): - mock_load_instance_using_id.return_value = _instance() +def test_eval_passes_only_when_every_instance_resolves(tmp_path): + task = _task(tmp_path, instance_id_list=["django__django-11099"]) + resolved = BotRunResult(status=True, result="resolved", error=None) - tasks = SweBenchVerifiedTask.from_config( - {"instance_id": "django__django-1", "swebench_repo": None} - ) + with patch.object(SweBenchVerifiedTask_one, "setup"), \ + patch.object(SweBenchVerifiedTask_one, "check", return_value=resolved), \ + patch(f"{MODULE}.WritingBot") as writing_bot, \ + patch(f"{MODULE}.MemoryTool"): + writing_bot.return_value.run.return_value = BotRunResult( + status=True, result="patched", error=None + ) + outcome = task.eval(str(tmp_path / "memory"), "azure-openai/gpt-4o", str(tmp_path / "eval")) - mock_load_instance_using_id.assert_called_once_with("django__django-1") - assert len(tasks) == 1 - assert isinstance(tasks[0], SweBenchVerifiedTask) - assert tasks[0].instance == _instance() + assert outcome.passed + assert outcome.score == 1 + assert outcome.feedback == "All evaluations passed." @pytest.mark.unit -@patch(f"{MODULE}.load_instances_of_repo") -def test_from_config_falls_back_to_repo_filter_when_no_instance_id(mock_load_instances_of_repo): - mock_load_instances_of_repo.return_value = [_instance(), _instance()] +def test_eval_skips_the_harness_when_the_agent_itself_failed(tmp_path): + task = _task(tmp_path, instance_id_list=["django__django-11099"]) - tasks = SweBenchVerifiedTask.from_config({"swebench_repo": "django/django"}) + with patch.object(SweBenchVerifiedTask_one, "setup"), \ + patch.object(SweBenchVerifiedTask_one, "check") as check, \ + patch(f"{MODULE}.WritingBot") as writing_bot, \ + patch(f"{MODULE}.MemoryTool"), \ + patch(f"{MODULE}.ReadingBot") as reading_bot: + writing_bot.return_value.run.return_value = BotRunResult( + status=False, result=None, error="agent timed out" + ) + reading_bot.return_value.run.return_value = BotRunResult( + status=True, result="the agent never produced a patch", error=None + ) + outcome = task.eval(str(tmp_path / "memory"), "azure-openai/gpt-4o", str(tmp_path / "eval")) - mock_load_instances_of_repo.assert_called_once_with(repo="django/django") - assert len(tasks) == 2 - assert all(isinstance(t, SweBenchVerifiedTask) for t in tasks) + check.assert_not_called() + assert outcome.score == 0 @pytest.mark.unit -@patch(f"{MODULE}.load_instances_of_repo") -def test_from_config_handles_empty_dict_gracefully(mock_load_instances_of_repo): - mock_load_instances_of_repo.return_value = [_instance()] +def test_combined_feedback_falls_back_to_raw_results_when_the_bot_fails(tmp_path): + task = _task(tmp_path, instance_id_list=["django__django-11099"]) + results = [BotRunResult(status=False, result="not resolved", error="assertion failed")] - tasks = SweBenchVerifiedTask.from_config({}) + with patch(f"{MODULE}.ReadingBot", side_effect=RuntimeError("no model configured")): + feedback = task._combine_result_feedback(results, "azure-openai/gpt-4o", "/repo") - mock_load_instances_of_repo.assert_called_once_with(repo=None) - assert len(tasks) == 1 + assert "assertion failed" in feedback diff --git a/test/auto_memory/test_cli.py b/test/auto_memory/test_cli.py index 6fcf716..be912cf 100644 --- a/test/auto_memory/test_cli.py +++ b/test/auto_memory/test_cli.py @@ -1,147 +1,78 @@ """Unit tests for microbots.auto_memory.cli.""" -import os -import sys from pathlib import Path from unittest.mock import MagicMock, patch import pytest -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) - from microbots.auto_memory.cli import main, parse_args MODULE = "microbots.auto_memory.cli" -BASE_ARGS = ["--model", "azure-openai/gpt-4o"] -FAKE_WORKDIR = Path("/workdir") - @pytest.mark.unit def test_parse_args_defaults(): - args = parse_args(BASE_ARGS) + args = parse_args(["--model", "azure-openai/gpt-4o", "--task", "swebenchverified"]) assert args.model == "azure-openai/gpt-4o" - assert args.task is None - assert args.max_rounds == 5 - assert args.training_iterations == 10 - - -@pytest.mark.unit -def test_parse_args_accepts_known_task(): - args = parse_args(BASE_ARGS + ["--task", "swebenchverified"]) - assert args.task == "swebenchverified" - - -@pytest.mark.unit -def test_parse_args_rejects_unknown_task(): - with pytest.raises(SystemExit): - parse_args(BASE_ARGS + ["--task", "does-not-exist"]) - - -@pytest.mark.unit -def test_parse_args_workdir_defaults_to_none(): - args = parse_args(BASE_ARGS) - + assert args.max_rounds == 5 assert args.workdir is None @pytest.mark.unit -def test_parse_args_picks_up_explicit_workdir(): - args = parse_args(BASE_ARGS + ["--workdir", "/custom/workdir"]) +def test_main_builds_the_task_from_the_workdir_config_and_runs_the_loop(tmp_path): + workdir = tmp_path / "workdir" + workdir.mkdir() + (workdir / "task_config.yaml").write_text("repo: django/django") - assert args.workdir == "/custom/workdir" + task_cls = MagicMock() + with patch.dict(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": task_cls}, clear=True), \ + patch(f"{MODULE}.run") as mock_run: + main([ + "--model", "azure-openai/gpt-4o", + "--task", "swebenchverified", + "--workdir", str(workdir), + "--max-rounds", "2", + ]) -@pytest.mark.unit -@patch(f"{MODULE}.require_workdir") -@patch(f"{MODULE}.resolve_workdir") -@patch(f"{MODULE}.load_config", return_value={}) -@patch(f"{MODULE}.run") -def test_main_uses_explicit_workdir_over_resolve_workdir( - mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir -): - main(BASE_ARGS + ["--workdir", "/custom/workdir"]) - - mock_resolve_workdir.assert_not_called() - mock_require_workdir.assert_called_once_with(Path("/custom/workdir")) - mock_run.assert_called_once_with( - workdir=Path("/custom/workdir"), - model="azure-openai/gpt-4o", - task=None, - max_rounds=5, - training_iterations=10, - config={}, - ) + task_cls.assert_called_once_with(config_file=workdir / "task_config.yaml") + assert mock_run.call_args.kwargs["max_rounds"] == 2 + assert mock_run.call_args.kwargs["task"] is task_cls.return_value @pytest.mark.unit -@patch(f"{MODULE}.require_workdir") -@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR) -@patch(f"{MODULE}.load_config", return_value={}) -@patch(f"{MODULE}.run") -def test_main_falls_back_to_resolve_workdir_when_not_given( - mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir -): - main(BASE_ARGS) - - mock_resolve_workdir.assert_called_once_with() - mock_require_workdir.assert_called_once_with(FAKE_WORKDIR) - mock_run.assert_called_once_with( - workdir=FAKE_WORKDIR, - model="azure-openai/gpt-4o", - task=None, - max_rounds=5, - training_iterations=10, - config={}, - ) - +def test_main_prefers_an_explicit_config_file(tmp_path): + workdir = tmp_path / "workdir" + workdir.mkdir() + config_file = tmp_path / "elsewhere.yaml" + config_file.write_text("repo: django/django") -@pytest.mark.unit -@patch(f"{MODULE}.require_workdir") -@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR) -@patch(f"{MODULE}.run") -def test_main_calls_run_with_task_none_when_task_omitted(mock_run, mock_resolve_workdir, mock_require_workdir): - main(BASE_ARGS) + task_cls = MagicMock() - assert mock_run.call_args.kwargs["task"] is None + with patch.dict(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": task_cls}, clear=True), \ + patch(f"{MODULE}.run"): + main([ + "--model", "azure-openai/gpt-4o", + "--task", "swebenchverified", + "--workdir", str(workdir), + "--config-file", str(config_file), + ]) - -@pytest.mark.unit -@patch(f"{MODULE}.require_workdir") -@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR) -@patch(f"{MODULE}.load_config", return_value={}) -@patch(f"{MODULE}.run") -def test_main_calls_run_for_each_task_when_task_given( - mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir -): - fake_task = MagicMock() - mock_run.return_value = MagicMock(passed=True, rounds_run=1) - - with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_config=lambda task_args: [fake_task])}): - main(BASE_ARGS + ["--task", "swebenchverified"]) - - mock_run.assert_called_once_with( - workdir=FAKE_WORKDIR, - model="azure-openai/gpt-4o", - task=fake_task, - max_rounds=5, - training_iterations=10, - config={}, - ) + task_cls.assert_called_once_with(config_file=config_file) @pytest.mark.unit -@patch(f"{MODULE}.require_workdir") -@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR) -@patch(f"{MODULE}.load_config", return_value={}) -@patch(f"{MODULE}.run") -def test_main_runs_once_per_returned_task(mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir): - fake_tasks = [MagicMock(), MagicMock()] - mock_run.return_value = MagicMock(passed=False, rounds_run=5) - - with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_config=lambda task_args: fake_tasks)}): - main(BASE_ARGS + ["--task", "swebenchverified"]) - - assert mock_run.call_count == 2 +def test_main_fails_fast_when_no_config_file_exists(tmp_path): + workdir = tmp_path / "workdir" + workdir.mkdir() + + with patch.dict(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock()}, clear=True), \ + patch(f"{MODULE}.run"): + with pytest.raises(FileNotFoundError): + main([ + "--model", "azure-openai/gpt-4o", + "--task", "swebenchverified", + "--workdir", str(workdir), + ]) diff --git a/test/auto_memory/test_full_loop.py b/test/auto_memory/test_full_loop.py new file mode 100644 index 0000000..7fd108a --- /dev/null +++ b/test/auto_memory/test_full_loop.py @@ -0,0 +1,129 @@ +"""End-to-end test of the train <-> eval loop on one SWE-bench instance. + +Only the outside world is mocked: the SWE-bench dataset, git, the +evaluation harness, and the three bots. Everything inside +``microbots.auto_memory`` runs for real, so this exercises the whole +path a live run takes: + + cli.main + -> require_workdir (workdir laid out) + -> SweBenchVerified (config parsed, instance loaded) + -> orchestrator.run (training repo cloned) + -> round 1: eval -> harness says unresolved -> feedback -> training + -> round 2: eval -> harness says resolved -> loop returns + +The instance is unresolved on the first round and resolved on the +second, so both the failure/retrain branch and the success branch are +covered in a single run. +""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import yaml + +from microbots.auto_memory.cli import main +from microbots.auto_memory.eval.swebenchverified import ( + SweBenchInstance, + SweBenchVerifiedTask_one, +) +from microbots.MicroBot import BotRunResult + +SWE_MODULE = "microbots.auto_memory.eval.swebenchverified" + +INSTANCE = SweBenchInstance( + instance_id="django__django-11099", + repo="django/django", + base_commit="abc123", + problem_statement="UsernameValidator allows trailing newline in usernames", +) + + +@pytest.fixture +def workdir(tmp_path): + workdir = tmp_path / "workdir" + workdir.mkdir() + (workdir / "task_config.yaml").write_text( + yaml.safe_dump({"instance_id_list": [INSTANCE.instance_id]}) + ) + return workdir + + +@pytest.mark.integration +def test_full_loop_retrains_after_a_failed_round_then_passes(workdir): + training_bot = MagicMock() + training_bot.run.return_value = BotRunResult(status=True, result="notes written", error=None) + + eval_bot = MagicMock() + eval_bot.run.return_value = BotRunResult(status=True, result="patch applied", error=None) + + feedback_bot = MagicMock() + feedback_bot.run.return_value = BotRunResult( + status=True, result="Memory must cover the username validator regex.", error=None + ) + + # The agent gets it wrong the first round and right the second. + harness_verdicts = [ + BotRunResult(status=False, result="not resolved", error="FAILED test_validators.py"), + BotRunResult(status=True, result="resolved", error=None), + ] + + def fake_clone(url, repo_path): + Path(repo_path).mkdir(parents=True, exist_ok=True) + + def fake_setup(self, repo_path): + Path(repo_path).mkdir(parents=True, exist_ok=True) + + with patch(f"{SWE_MODULE}.load_instance_using_id", return_value=INSTANCE), \ + patch("microbots.auto_memory.orchestrator.clone_repo", side_effect=fake_clone) as clone, \ + patch.object(SweBenchVerifiedTask_one, "setup", fake_setup), \ + patch.object(SweBenchVerifiedTask_one, "check", side_effect=harness_verdicts), \ + patch(f"{SWE_MODULE}.WritingBot", return_value=eval_bot), \ + patch(f"{SWE_MODULE}.ReadingBot", return_value=feedback_bot), \ + patch(f"{SWE_MODULE}.MemoryTool"), \ + patch("microbots.auto_memory.training.runner.ReadingBot", return_value=training_bot), \ + patch("microbots.auto_memory.training.runner.MemoryTool"): + main([ + "--model", "azure-openai/gpt-4o", + "--task", "swebenchverified", + "--workdir", str(workdir), + "--max-rounds", "3", + ]) + + # The training repo is derived from the instance, not from the config. + assert clone.call_args.args[0] == "https://github.com/django/django.git" + + # Two rounds ran, and only the failing one triggered retraining. + assert eval_bot.run.call_count == 2 + training_bot.run.assert_called_once() + assert not (workdir / "rounds" / "round_3").exists() + + # The failed round's feedback is what the training agent was given. + training_prompt = training_bot.run.call_args.args[0] + assert "Memory must cover the username validator regex." in training_prompt + + # Round 1 recorded a failure, round 2 a pass. + round_1 = _result(workdir, 1) + assert round_1["passed"] is False + assert round_1["score"] == 0 + + round_2 = _result(workdir, 2) + assert round_2["passed"] is True + assert round_2["score"] == 1 + + # Each round kept the memory it started from, and the per-instance log. + for round_num in (1, 2): + round_path = workdir / "rounds" / f"round_{round_num}" + assert (round_path / "starting_memory_snapshot").is_dir() + assert ( + round_path / "eval" / "logs" / f"{INSTANCE.instance_id}_log.txt" + ).is_file() + + +def _result(workdir: Path, round_num: int) -> dict: + import json + + return json.loads( + (workdir / "rounds" / f"round_{round_num}" / "eval" / "result.json").read_text() + ) diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py index ba193be..b147d58 100644 --- a/test/auto_memory/test_orchestrator.py +++ b/test/auto_memory/test_orchestrator.py @@ -1,521 +1,116 @@ -"""Unit tests for microbots.auto_memory.orchestrator.""" +"""Unit tests for microbots.auto_memory.orchestrator. + +The training bot is mocked out; these tests only cover how the loop +sequences rounds, reacts to outcomes, and records results. +""" import json -import os -import sys -from pathlib import Path -from unittest.mock import MagicMock, call, patch +from unittest.mock import MagicMock, patch import pytest -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) - +from microbots.auto_memory.evalTask import EvalOutcome from microbots.auto_memory.orchestrator import ( - LoopResult, - clone_repo, - run, run_train_eval_loop, - run_training_loop, write_eval_result, ) -from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome -from microbots.auto_memory.workdir import eval_result_path, memory_dir, round_memory_dir MODULE = "microbots.auto_memory.orchestrator" -def _make_outcome(passed: bool, reason: str = "reason") -> EvalOutcome: - return EvalOutcome( - passed=passed, - output="agent output", - result=CallbackResult(passed=passed, reason=reason), - ) - - -def _touch(path: str) -> str: - Path(path).write_text("log contents") - return path +def _outcome(passed: bool, score: float = 0.0, feedback: str = "needs work") -> EvalOutcome: + return EvalOutcome(passed=passed, score=score, feedback=feedback) -def _make_task() -> MagicMock: - """A MagicMock task with a real-ish task_id/build_result, for round tests.""" +def _task(*outcomes: EvalOutcome) -> MagicMock: task = MagicMock() - task.task_id = "task-1" - task.build_result.side_effect = lambda outcome: { - "passed": outcome.result.passed, - "reason": outcome.result.reason, - } + task.eval.side_effect = list(outcomes) return task -@pytest.mark.unit -@pytest.mark.parametrize("max_rounds", [0, -1]) -def test_loop_raises_for_non_positive_max_rounds(max_rounds): - task = _make_task() - - with pytest.raises(ValueError, match="max_rounds must be >= 1"): - run_train_eval_loop("/repo", "/eval_repo", Path("/workdir"), "azure-openai/gpt-4o", task, max_rounds=max_rounds) - - task.run.assert_not_called() +@pytest.fixture +def workdir(tmp_path): + (tmp_path / "memory").mkdir() + return tmp_path @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training_loop") -def test_loop_returns_immediately_when_first_round_passes(mock_run_training_loop, tmp_path): - task = _make_task() - task.run.return_value = _make_outcome(passed=True) - - result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) - - assert isinstance(result, LoopResult) - assert result.passed is True - assert result.rounds_run == 1 - assert task.run.call_count == 1 - task.build_feedback.assert_not_called() - mock_run_training_loop.assert_not_called() - +def test_write_eval_result_serializes_the_outcome(tmp_path): + write_eval_result(tmp_path, _outcome(True, score=1.0, feedback="all good")) -@pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training_loop") -def test_loop_retrains_and_continues_on_failure_then_passes(mock_run_training_loop, tmp_path): - task = _make_task() - task.run.side_effect = [ - _make_outcome(passed=False), - _make_outcome(passed=True), - ] - task.build_feedback.return_value = "feedback text" - - result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) - - assert result.passed is True - assert result.rounds_run == 2 - task.build_feedback.assert_called_once() - mock_run_training_loop.assert_called_once_with( - repo_path="/repo", - feedback="feedback text", - memory_dir=str(round_memory_dir(tmp_path, 1, instance_id="task-1")), - model="azure-openai/gpt-4o", - iterations=10, - ) - - -@pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training_loop") -def test_loop_exhausts_max_rounds_without_passing(mock_run_training_loop, tmp_path): - task = _make_task() - task.run.side_effect = [ - _make_outcome(passed=False) - for i in range(3) - ] - task.build_feedback.return_value = "feedback text" - - result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=3) - - assert result.passed is False - assert result.rounds_run == 3 - assert len(result.outcomes) == 3 - assert result.final_outcome is result.outcomes[-1] - assert task.build_feedback.call_count == 3 - assert mock_run_training_loop.call_count == 3 + assert json.loads((tmp_path / "result.json").read_text()) == { + "passed": True, + "score": 1.0, + "feedback": "all good", + } @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training_loop") -def test_log_path_persists_after_passing_round(mock_run_training_loop, tmp_path): - log_path = _touch(str(tmp_path / "round1.log")) - task = _make_task() - task.run.return_value = _make_outcome(passed=True) +def test_loop_stops_on_the_first_passing_round(workdir): + task = _task(_outcome(True, score=1.0)) - run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + with patch(f"{MODULE}.run_training") as mock_training: + result = run_train_eval_loop("/repo", workdir, "azure-openai/gpt-4o", task, max_rounds=3) - assert Path(log_path).exists() + assert result.passed + assert result.rounds_run == 1 + assert task.eval.call_count == 1 + mock_training.assert_not_called() @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training_loop") -def test_log_path_persists_after_failing_round(mock_run_training_loop, tmp_path): - log1 = _touch(str(tmp_path / "round1.log")) - log2 = _touch(str(tmp_path / "round2.log")) - task = _make_task() - task.run.side_effect = [ - _make_outcome(passed=False), - _make_outcome(passed=True), - ] - task.build_feedback.return_value = "feedback text" +def test_loop_retrains_with_the_round_feedback_then_passes(workdir): + task = _task(_outcome(False, feedback="cover the settings module"), _outcome(True, score=1.0)) - run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + with patch(f"{MODULE}.run_training") as mock_training: + result = run_train_eval_loop("/repo", workdir, "azure-openai/gpt-4o", task, max_rounds=3) - assert Path(log1).exists() - assert Path(log2).exists() - - -@pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training_loop") -def test_build_feedback_exception_does_not_crash_loop(mock_run_training_loop, tmp_path): - log1 = _touch(str(tmp_path / "round1.log")) - task = _make_task() - task.run.side_effect = [ - _make_outcome(passed=False), - _make_outcome(passed=True), - ] - task.build_feedback.side_effect = RuntimeError("analysis bot crashed") - - result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) - - assert result.passed is True + assert result.passed assert result.rounds_run == 2 - mock_run_training_loop.assert_not_called() - assert Path(log1).exists() + mock_training.assert_called_once() + assert mock_training.call_args.kwargs["feedback"] == "cover the settings module" @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training_loop") -def test_loop_forwards_training_iterations_to_run_training_loop(mock_run_training_loop, tmp_path): - task = _make_task() - task.run.side_effect = [ - _make_outcome(passed=False), - _make_outcome(passed=True), - ] - task.build_feedback.return_value = "feedback text" - - run_train_eval_loop( - "/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5, training_iterations=4 - ) - - mock_run_training_loop.assert_called_once_with( - repo_path="/repo", - feedback="feedback text", - memory_dir=str(round_memory_dir(tmp_path, 1, instance_id="task-1")), - model="azure-openai/gpt-4o", - iterations=4, - ) +def test_loop_gives_up_after_max_rounds(workdir): + task = _task(_outcome(False), _outcome(False)) + with patch(f"{MODULE}.run_training"): + result = run_train_eval_loop("/repo", workdir, "azure-openai/gpt-4o", task, max_rounds=2) -@pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training_loop") -def test_run_training_exception_does_not_crash_loop(mock_run_training_loop, tmp_path): - log1 = _touch(str(tmp_path / "round1.log")) - task = _make_task() - task.run.side_effect = [ - _make_outcome(passed=False), - _make_outcome(passed=True), - ] - task.build_feedback.return_value = "feedback text" - mock_run_training_loop.side_effect = RuntimeError("training crashed") - - result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) - - assert result.passed is True + assert not result.passed assert result.rounds_run == 2 - assert Path(log1).exists() - - -@pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") -def test_run_training_loop_calls_run_training_ten_times_by_default(mock_run_training): - run_training_loop(repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o") - - assert mock_run_training.call_count == 10 - mock_run_training.assert_called_with( - repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o" - ) - - -@pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") -def test_run_training_loop_calls_run_training_n_times(mock_run_training): - run_training_loop( - repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=3 - ) - - assert mock_run_training.call_count == 3 - mock_run_training.assert_called_with( - repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o" - ) - - -@pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") -def test_run_training_loop_reuses_same_memory_dir_each_pass(mock_run_training): - run_training_loop( - repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=4 - ) - - memory_dirs = {call.kwargs["memory_dir"] for call in mock_run_training.call_args_list} - assert memory_dirs == {"/memory"} + assert len(result.outcomes) == 2 @pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_clone_repo_clones_when_missing(mock_run, tmp_path): - repo_path = tmp_path / "repo" - - clone_repo("https://example.com/repo.git", repo_path) - - mock_run.assert_called_once_with( - ["git", "clone", "https://example.com/repo.git", str(repo_path)], check=True - ) - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_clone_repo_is_noop_when_origin_matches(mock_run, tmp_path): - repo_path = tmp_path / "repo" - repo_path.mkdir() - mock_run.return_value = MagicMock(returncode=0, stdout="https://example.com/repo.git\n") - - clone_repo("https://example.com/repo.git", repo_path) - - mock_run.assert_called_once_with( - ["git", "remote", "get-url", "origin"], - cwd=repo_path, capture_output=True, text=True, - ) - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_clone_repo_removes_and_reclones_when_origin_mismatched(mock_run, tmp_path): - repo_path = tmp_path / "repo" - repo_path.mkdir() - (repo_path / "stale_marker.txt").write_text("leftover from a different repo") - mock_run.return_value = MagicMock(returncode=0, stdout="https://example.com/other-repo.git\n") - - clone_repo("https://example.com/repo.git", repo_path) - - assert not (repo_path / "stale_marker.txt").exists() - mock_run.assert_called_with( - ["git", "clone", "https://example.com/repo.git", str(repo_path)], check=True - ) - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_clone_repo_removes_and_reclones_when_repo_path_not_a_git_repo(mock_run, tmp_path): - repo_path = tmp_path / "repo" - repo_path.mkdir() - mock_run.return_value = MagicMock(returncode=128, stdout="") - - clone_repo("https://example.com/repo.git", repo_path) - - assert not repo_path.exists() - mock_run.assert_called_with( - ["git", "clone", "https://example.com/repo.git", str(repo_path)], check=True - ) - - -@pytest.mark.unit -def test_write_eval_result_writes_task_build_result_as_json(tmp_path): +def test_a_raising_eval_is_recorded_and_the_loop_continues(workdir): task = MagicMock() - task.task_id = "django__django-1" - task.build_result.return_value = {"passed": True, "reason": "resolved"} - outcome = _make_outcome(passed=True) - - write_eval_result(tmp_path, 2, task, outcome) - - result_path = eval_result_path(tmp_path, 2, "django__django-1") - assert json.loads(result_path.read_text()) == {"passed": True, "reason": "resolved"} - task.build_result.assert_called_once_with(outcome) + task.eval.side_effect = [RuntimeError("harness exploded"), _outcome(True, score=1.0)] + with patch(f"{MODULE}.run_training"): + result = run_train_eval_loop("/repo", workdir, "azure-openai/gpt-4o", task, max_rounds=2) -@pytest.mark.unit -def test_write_eval_result_creates_missing_parent_dirs(tmp_path): - task = MagicMock() - task.task_id = "some-task" - task.build_result.return_value = {"passed": False, "reason": "nope"} - outcome = _make_outcome(passed=False) - - write_eval_result(tmp_path, 1, task, outcome) - - assert eval_result_path(tmp_path, 1, "some-task").exists() - - -@pytest.mark.unit -def test_loop_writes_eval_result_for_every_round(tmp_path): - task = _make_task() - task.run.side_effect = [ - _make_outcome(passed=False), - _make_outcome(passed=True), - ] - task.build_feedback.return_value = "feedback text" - - with patch(f"{MODULE}.run_training_loop"): - run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) - - assert eval_result_path(tmp_path, 1, "task-1").exists() - assert eval_result_path(tmp_path, 2, "task-1").exists() - assert json.loads(eval_result_path(tmp_path, 2, "task-1").read_text()) == { - "passed": True, - "reason": "reason", - } - - -@pytest.mark.unit -@patch(f"{MODULE}.clone_repo") -@patch(f"{MODULE}.run_training_loop") -def test_run_calls_run_training_loop_when_task_is_none(mock_run_training_loop, mock_clone_repo, tmp_path): - result = run( - workdir=tmp_path, - model="azure-openai/gpt-4o", - task=None, - training_iterations=2, - config={"repo": "https://example.com/repo.git"}, - ) - - mock_run_training_loop.assert_called_once_with( - repo_path=str(tmp_path / "repo"), - feedback="", - memory_dir=str(round_memory_dir(tmp_path, 1)), - model="azure-openai/gpt-4o", - iterations=2, - ) - assert result is None - - -@pytest.mark.unit -@patch(f"{MODULE}.clone_repo") -@patch(f"{MODULE}.run_train_eval_loop") -def test_run_calls_run_train_eval_loop_when_task_given(mock_run_train_eval_loop, mock_clone_repo, tmp_path): - fake_task = MagicMock() - mock_run_train_eval_loop.return_value = "loop-result" - - result = run( - workdir=tmp_path, - model="azure-openai/gpt-4o", - task=fake_task, - max_rounds=3, - training_iterations=2, - config={"repo": "https://example.com/repo.git"}, - ) - - mock_run_train_eval_loop.assert_called_once_with( - training_repo_path=str(tmp_path / "repo"), - eval_repo_path=str(tmp_path / "eval_repo"), - workdir=tmp_path, - model="azure-openai/gpt-4o", - task=fake_task, - max_rounds=3, - training_iterations=2, - ) - assert result == "loop-result" + assert result.passed + assert result.outcomes[0].score == -1 + assert "harness exploded" in result.outcomes[0].feedback @pytest.mark.unit -@patch(f"{MODULE}.clone_repo") -@patch(f"{MODULE}.run_train_eval_loop") -@patch(f"{MODULE}.run_training_loop") -def test_run_does_not_call_eval_loop_when_task_is_none( - mock_run_training_loop, mock_run_train_eval_loop, mock_clone_repo, tmp_path -): - run( - workdir=tmp_path, - model="azure-openai/gpt-4o", - task=None, - config={"repo": "https://example.com/repo.git"}, - ) - - mock_run_train_eval_loop.assert_not_called() - - -@pytest.mark.unit -@patch(f"{MODULE}.clone_repo") -@patch(f"{MODULE}.run_train_eval_loop") -@patch(f"{MODULE}.run_training_loop") -def test_run_does_not_call_training_loop_when_task_given( - mock_run_training_loop, mock_run_train_eval_loop, mock_clone_repo, tmp_path -): - run( - workdir=tmp_path, - model="azure-openai/gpt-4o", - task=MagicMock(), - config={"repo": "https://example.com/repo.git"}, - ) - - mock_run_training_loop.assert_not_called() - - -@pytest.mark.unit -@patch(f"{MODULE}.clone_repo") -@patch(f"{MODULE}.run_training_loop") -def test_run_clones_repo_from_config_when_repo_url_given(mock_run_training_loop, mock_clone_repo, tmp_path): - (tmp_path / "config.yaml").write_text("repo: https://example.com/repo.git\n") - - run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) - - mock_clone_repo.assert_called_once_with("https://example.com/repo.git", tmp_path / "repo") - - -@pytest.mark.unit -@patch(f"{MODULE}.clone_repo") -@patch(f"{MODULE}.run_training_loop") -def test_run_raises_when_config_has_no_repo(mock_run_training_loop, mock_clone_repo, tmp_path): - with pytest.raises(ValueError, match="repo"): - run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None, config={}) - - mock_clone_repo.assert_not_called() - mock_run_training_loop.assert_not_called() - - -@pytest.mark.unit -@patch(f"{MODULE}.clone_repo") -@patch(f"{MODULE}.run_training_loop") -def test_run_promotes_round1_memory_to_top_level_for_train_only_mode(mock_run_training_loop, mock_clone_repo, tmp_path): - def fake_train(repo_path, feedback, memory_dir, model, iterations=1): - Path(memory_dir, "notes.md").write_text("learned something") - - mock_run_training_loop.side_effect = fake_train - - run( - workdir=tmp_path, - model="azure-openai/gpt-4o", - task=None, - config={"repo": "https://example.com/repo.git"}, - ) - - assert (memory_dir(tmp_path) / "notes.md").read_text() == "learned something" - - -@pytest.mark.unit -@patch(f"{MODULE}.clone_repo") -@patch(f"{MODULE}.run_training_loop") -def test_run_preserves_original_memory_as_a_seed_snapshot(mock_run_training_loop, mock_clone_repo, tmp_path): - memory_dir(tmp_path).mkdir(parents=True) - (memory_dir(tmp_path) / "notes.md").write_text("original seed") - - def fake_train(repo_path, feedback, memory_dir, model, iterations=1): - Path(memory_dir, "notes.md").write_text("overwritten by training") - - mock_run_training_loop.side_effect = fake_train - - run( - workdir=tmp_path, - model="azure-openai/gpt-4o", - task=None, - config={"repo": "https://example.com/repo.git"}, - ) +def test_each_round_records_its_result_and_memory_snapshot(workdir): + (workdir / "memory" / "notes.md").write_text("what I know") + task = _task(_outcome(False), _outcome(False)) - assert (memory_dir(tmp_path) / "notes.md").read_text() == "overwritten by training" - assert (tmp_path / "memory_seed" / "notes.md").read_text() == "original seed" + with patch(f"{MODULE}.run_training"): + run_train_eval_loop("/repo", workdir, "azure-openai/gpt-4o", task, max_rounds=2) + for round_num in (1, 2): + round_path = workdir / "rounds" / f"round_{round_num}" + assert (round_path / "eval" / "result.json").is_file() + assert (round_path / "starting_memory_snapshot" / "notes.md").is_file() @pytest.mark.unit -@patch(f"{MODULE}.run_training_loop") -def test_loop_carries_memory_forward_between_rounds(mock_run_training_loop, tmp_path): - seen_memory_dirs = [] - - def fake_run(repo_path, memory_dir, model, log_path): - round_num = len(seen_memory_dirs) + 1 - if round_num == 2: - # Round 2 should start with whatever round 1 saved. - assert (Path(memory_dir) / "notes.md").read_text() == "round 1 progress" - seen_memory_dirs.append(memory_dir) - Path(memory_dir, "notes.md").write_text(f"round {round_num} progress") - return _make_outcome(passed=round_num == 2) - - task = _make_task() - task.run.side_effect = fake_run - task.build_feedback.return_value = "feedback text" - - run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) - - assert (memory_dir(tmp_path) / "notes.md").read_text() == "round 2 progress" +def test_max_rounds_must_be_at_least_one(workdir): + with pytest.raises(ValueError, match="max_rounds"): + run_train_eval_loop("/repo", workdir, "azure-openai/gpt-4o", MagicMock(), max_rounds=0) diff --git a/test/auto_memory/test_task.py b/test/auto_memory/test_task.py deleted file mode 100644 index e04d0e9..0000000 --- a/test/auto_memory/test_task.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Unit tests for microbots.auto_memory.evalTask.""" - -import os -import sys - -import pytest - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) - -from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome, EvalTask - - -class _RunOnlyTask(EvalTask): - """A task that overrides only run()/build_feedback(), never touching the optional hooks.""" - - @classmethod - def from_config(cls, task_args): - return [cls()] - - def run(self, repo_path, memory_dir, model, log_path): - return EvalOutcome( - passed=True, - output="custom output", - result=None, - ) - - def build_feedback(self, outcome, repo_path, model, log_path): - return "feedback text" - - -@pytest.mark.unit -def test_run_is_abstract(): - with pytest.raises(TypeError): - EvalTask() - - -@pytest.mark.unit -def test_build_feedback_is_abstract(): - class _MissingBuildFeedback(EvalTask): - @classmethod - def from_config(cls, task_args): - return [cls()] - - def run(self, repo_path, memory_dir, model, log_path): - raise NotImplementedError - - with pytest.raises(TypeError): - _MissingBuildFeedback() - - -@pytest.mark.unit -def test_from_config_is_abstract(): - class _MissingFromConfig(EvalTask): - def run(self, repo_path, memory_dir, model, log_path): - raise NotImplementedError - - def build_feedback(self, outcome, repo_path, model, log_path): - raise NotImplementedError - - with pytest.raises(TypeError): - _MissingFromConfig() - - -@pytest.mark.unit -def test_from_config_default_body_raises_not_implemented_error(): - with pytest.raises(NotImplementedError): - EvalTask.from_config({}) - - -@pytest.mark.unit -def test_subclass_overriding_only_run_is_instantiable(): - task = _RunOnlyTask() - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", "/log") - - assert outcome.passed is True - assert outcome.output == "custom output" - - -@pytest.mark.unit -def test_default_setup_is_a_noop(): - # Should not raise. - _RunOnlyTask().setup("/repo") - - -@pytest.mark.unit -def test_default_teardown_is_a_noop(): - # Should not raise. - _RunOnlyTask().teardown("/repo") - - -@pytest.mark.unit -def test_default_build_prompt_returns_empty_string(): - assert _RunOnlyTask().build_prompt() == "" - - -@pytest.mark.unit -def test_default_check_passes_by_default(): - result = _RunOnlyTask().check("/repo", "output", "/log") - - assert isinstance(result, CallbackResult) - assert result.passed is True - - -@pytest.mark.unit -def test_default_task_id_is_class_name(): - assert _RunOnlyTask().task_id == "_RunOnlyTask" - - -@pytest.mark.unit -def test_default_build_result_returns_passed_and_reason(): - outcome = EvalOutcome( - passed=False, - output="agent output", - result=CallbackResult(passed=False, reason="check failed"), - ) - - assert _RunOnlyTask().build_result(outcome) == {"passed": False, "reason": "check failed"} diff --git a/test/auto_memory/test_task_registry.py b/test/auto_memory/test_task_registry.py index 8de0d48..f761331 100644 --- a/test/auto_memory/test_task_registry.py +++ b/test/auto_memory/test_task_registry.py @@ -1,50 +1,29 @@ """Unit tests for microbots.auto_memory.task_registry.""" -import os -import sys -from types import SimpleNamespace -from unittest.mock import MagicMock, patch +from pathlib import Path import pytest -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) +from microbots.auto_memory.evalTask import EvalOutcome, EvalTask +from microbots.auto_memory.task_registry import ( + TASK_REGISTRY, + discover_tasks, + register_task, +) -from microbots.auto_memory.evalTask import EvalTask -from microbots.auto_memory.task_registry import TASK_REGISTRY, create_task, discover_tasks, register_task -MODULE_PATH = "microbots.auto_memory.task_registry" +class _StubTask(EvalTask): + """Minimal concrete task, enough to be registrable.""" + def parse_config(self, config_file: Path) -> None: + self._repo_url = "https://github.com/acme/widget.git" -class _DummyTask(EvalTask): - def __init__(self, value=None): - self.value = value - - @classmethod - def from_config(cls, task_args): - return [cls(**task_args)] - - def setup(self, repo_path): - pass - - def build_prompt(self): - return "prompt" - - def check(self, output): - pass - - def teardown(self, repo_path): - pass - - def build_feedback(self, outcome, repo_path, model, log_path): - return "feedback" - - def run(self, repo_path, memory_dir, model): - return super().run(repo_path, memory_dir, model) + def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: + return EvalOutcome(passed=True, score=1.0, feedback="ok") @pytest.fixture(autouse=True) -def _clean_registry(): - """Snapshot/restore TASK_REGISTRY so tests don't leak state into each other.""" +def _restore_registry(): original = dict(TASK_REGISTRY) yield TASK_REGISTRY.clear() @@ -52,60 +31,33 @@ def _clean_registry(): @pytest.mark.unit -def test_register_task_adds_class_to_registry(): - register_task("dummy")(_DummyTask) +def test_register_task_stores_the_class_under_its_name(): + register_task("stub")(_StubTask) - assert TASK_REGISTRY["dummy"] is _DummyTask + assert TASK_REGISTRY["stub"] is _StubTask @pytest.mark.unit -def test_register_task_returns_class_unchanged(): - decorated = register_task("dummy")(_DummyTask) +def test_registering_the_same_class_twice_is_allowed(): + register_task("stub")(_StubTask) + register_task("stub")(_StubTask) - assert decorated is _DummyTask + assert TASK_REGISTRY["stub"] is _StubTask @pytest.mark.unit -def test_create_task_constructs_registered_task_with_kwargs(): - register_task("dummy")(_DummyTask) - - task = create_task("dummy", value=42) - - assert isinstance(task, _DummyTask) - assert task.value == 42 +def test_registering_a_second_class_under_one_name_raises(): + class _OtherTask(_StubTask): + pass + register_task("stub")(_StubTask) -@pytest.mark.unit -def test_create_task_raises_for_unknown_name(): - with pytest.raises(ValueError, match="Unknown task 'nonexistent'"): - create_task("nonexistent") + with pytest.raises(ValueError, match="already registered"): + register_task("stub")(_OtherTask) @pytest.mark.unit -def test_discover_tasks_registers_swebenchverified(): - """Non-destructive: confirms discover_tasks() works against the real package.""" +def test_discover_tasks_registers_the_shipped_eval_tasks(): discover_tasks() assert "swebenchverified" in TASK_REGISTRY - - -@pytest.mark.unit -@patch(f"{MODULE_PATH}.importlib.import_module") -@patch(f"{MODULE_PATH}.pkgutil.iter_modules") -def test_discover_tasks_imports_every_module_found_in_package(mock_iter_modules, mock_import_module): - fake_package = MagicMock() - fake_package.__path__ = ["/fake/path"] - mock_import_module.side_effect = ( - lambda name: fake_package if name == "fake.pkg" else MagicMock() - ) - mock_iter_modules.return_value = [ - SimpleNamespace(name="task_a"), - SimpleNamespace(name="task_b"), - ] - - discover_tasks(package_name="fake.pkg") - - mock_iter_modules.assert_called_once_with(["/fake/path"]) - mock_import_module.assert_any_call("fake.pkg") - mock_import_module.assert_any_call("fake.pkg.task_a") - mock_import_module.assert_any_call("fake.pkg.task_b") diff --git a/test/auto_memory/test_workdir.py b/test/auto_memory/test_workdir.py index 4c87baa..29a3a56 100644 --- a/test/auto_memory/test_workdir.py +++ b/test/auto_memory/test_workdir.py @@ -1,277 +1,99 @@ -"""Unit tests for microbots.auto_memory.workdir.""" +"""Unit tests for microbots.auto_memory.workdir. -import os -import sys +These are pure filesystem-layout tests; nothing here touches git, an +LLM, or Docker. +""" import pytest -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) - from microbots.auto_memory.workdir import ( - CONFIG_FILENAME, - eval_dir, - eval_log_path, - eval_patch_path, - eval_repo_dir, - eval_result_path, - load_config, - load_round_memory, + config_path, + get_eval_dir, memory_dir, repo_dir, require_workdir, resolve_workdir, round_dir, - round_log_path, - round_memory_dir, - save_round_memory, - snapshot_seed_memory, + take_memory_snapshot, ) @pytest.mark.unit -def test_load_config_returns_empty_dict_when_file_missing(tmp_path): - assert load_config(tmp_path) == {} - - -@pytest.mark.unit -def test_load_config_returns_empty_dict_when_file_empty(tmp_path): - (tmp_path / CONFIG_FILENAME).write_text("") - - assert load_config(tmp_path) == {} - - -@pytest.mark.unit -def test_load_config_parses_yaml_contents(tmp_path): - (tmp_path / CONFIG_FILENAME).write_text("repo: https://example.com/repo.git\ntask: swebenchverified\n") - - assert load_config(tmp_path) == { - "repo": "https://example.com/repo.git", - "task": "swebenchverified", - } - - -@pytest.mark.unit -def test_load_round_memory_creates_empty_dir_when_no_top_level_memory(tmp_path): - result = load_round_memory(tmp_path, 1) - - assert result == round_memory_dir(tmp_path, 1) - assert result.is_dir() - assert list(result.iterdir()) == [] - - -@pytest.mark.unit -def test_load_round_memory_copies_top_level_memory_into_round(tmp_path): - top_memory = memory_dir(tmp_path) - top_memory.mkdir(parents=True) - (top_memory / "notes.md").write_text("prior findings") - - result = load_round_memory(tmp_path, 2) - - assert (result / "notes.md").read_text() == "prior findings" - - -@pytest.mark.unit -def test_load_round_memory_discards_stale_files_left_in_round_dir(tmp_path): - # Simulate a previous crashed/re-run attempt at this same round that - # left behind a file no longer present in top-level memory. - round_memory = round_memory_dir(tmp_path, 1) - round_memory.mkdir(parents=True) - (round_memory / "stale.md").write_text("leftover from a crashed attempt") - - top_memory = memory_dir(tmp_path) - top_memory.mkdir(parents=True) - (top_memory / "notes.md").write_text("current memory") - - result = load_round_memory(tmp_path, 1) - - assert (result / "notes.md").read_text() == "current memory" - assert not (result / "stale.md").exists() - - -@pytest.mark.unit -def test_save_round_memory_creates_empty_dir_when_no_round_memory(tmp_path): - result = save_round_memory(tmp_path, 1) - - assert result == memory_dir(tmp_path) - assert result.is_dir() - assert list(result.iterdir()) == [] - - -@pytest.mark.unit -def test_save_round_memory_copies_round_memory_to_top_level(tmp_path): - round_memory = round_memory_dir(tmp_path, 3) - round_memory.mkdir(parents=True) - (round_memory / "learned.md").write_text("new insight") - - result = save_round_memory(tmp_path, 3) - - assert (result / "learned.md").read_text() == "new insight" - - -@pytest.mark.unit -def test_save_round_memory_overwrites_stale_top_level_files(tmp_path): - top_memory = memory_dir(tmp_path) - top_memory.mkdir(parents=True) - (top_memory / "notes.md").write_text("old") - - round_memory = round_memory_dir(tmp_path, 1) - round_memory.mkdir(parents=True) - (round_memory / "notes.md").write_text("new") - - save_round_memory(tmp_path, 1) - - assert (top_memory / "notes.md").read_text() == "new" - - -@pytest.mark.unit -def test_save_round_memory_propagates_deletions_to_top_level(tmp_path): - # The agent deleted a file during this round (e.g. via `memory - # delete`); the top level shouldn't resurrect it from a prior save. - top_memory = memory_dir(tmp_path) - top_memory.mkdir(parents=True) - (top_memory / "stale.md").write_text("no longer relevant") - - round_memory = round_memory_dir(tmp_path, 1) - round_memory.mkdir(parents=True) - (round_memory / "notes.md").write_text("kept") - - save_round_memory(tmp_path, 1) - - assert not (top_memory / "stale.md").exists() - assert (top_memory / "notes.md").read_text() == "kept" - - -@pytest.mark.unit -def test_snapshot_seed_memory_creates_empty_dir_when_no_top_level_memory(tmp_path): - result = snapshot_seed_memory(tmp_path) - - assert result == tmp_path / "memory_seed" - assert result.is_dir() - assert list(result.iterdir()) == [] - - -@pytest.mark.unit -def test_snapshot_seed_memory_copies_current_top_level_memory(tmp_path): - top_memory = memory_dir(tmp_path) - top_memory.mkdir(parents=True) - (top_memory / "notes.md").write_text("original seed") - - result = snapshot_seed_memory(tmp_path) - - assert (result / "notes.md").read_text() == "original seed" - - -@pytest.mark.unit -def test_snapshot_seed_memory_is_a_noop_once_a_snapshot_exists(tmp_path): - top_memory = memory_dir(tmp_path) - top_memory.mkdir(parents=True) - (top_memory / "notes.md").write_text("original seed") - snapshot_seed_memory(tmp_path) - - # Mutate top-level memory as later rounds/instances would. - (top_memory / "notes.md").write_text("overwritten by later training") - - result = snapshot_seed_memory(tmp_path) - - assert (result / "notes.md").read_text() == "original seed" - - -@pytest.mark.unit -def test_resolve_workdir_defaults_to_cwd(monkeypatch, tmp_path): - monkeypatch.chdir(tmp_path) - - assert resolve_workdir() == tmp_path / "workdir" - - -@pytest.mark.unit -def test_resolve_workdir_uses_given_base(tmp_path): +def test_layout_paths_hang_off_the_workdir(tmp_path): assert resolve_workdir(tmp_path) == tmp_path / "workdir" - - -@pytest.mark.unit -def test_require_workdir_raises_when_missing(tmp_path): - missing = tmp_path / "nope" - - with pytest.raises(FileNotFoundError): - require_workdir(missing) - - -@pytest.mark.unit -def test_require_workdir_passes_when_present(tmp_path): - require_workdir(tmp_path) - - -@pytest.mark.unit -def test_repo_dir_returns_workdir_repo(tmp_path): + assert config_path(tmp_path) == tmp_path / "task_config.yaml" assert repo_dir(tmp_path) == tmp_path / "repo" + assert memory_dir(tmp_path) == tmp_path / "memory" + assert round_dir(tmp_path, 2) == tmp_path / "rounds" / "round_2" + assert get_eval_dir(tmp_path, 2) == tmp_path / "rounds" / "round_2" / "eval" @pytest.mark.unit -def test_eval_repo_dir_returns_workdir_eval_repo(tmp_path): - assert eval_repo_dir(tmp_path) == tmp_path / "eval_repo" +def test_require_workdir_creates_memory_dir_for_a_new_workdir(tmp_path): + workdir = tmp_path / "workdir" + require_workdir(workdir) -@pytest.mark.unit -def test_round_dir_creates_directory_when_requested(tmp_path): - path = round_dir(tmp_path, 1, create=True) - - assert path == tmp_path / "rounds" / "round_1" - assert path.is_dir() + assert memory_dir(workdir).is_dir() @pytest.mark.unit -def test_round_dir_does_not_create_directory_by_default(tmp_path): - path = round_dir(tmp_path, 1) - - assert path == tmp_path / "rounds" / "round_1" - assert not path.exists() +def test_require_workdir_archives_previous_run_and_carries_state_over(tmp_path): + workdir = tmp_path / "workdir" + (workdir / "memory").mkdir(parents=True) + (workdir / "memory" / "notes.md").write_text("learned") + (workdir / "repo").mkdir() + (workdir / "repo" / "code.py").write_text("x = 1") + (workdir / "task_config.yaml").write_text("repo: acme/widget") + (workdir / "rounds" / "round_1").mkdir(parents=True) + require_workdir(workdir) -@pytest.mark.unit -def test_round_dir_uses_per_instance_dir_when_instance_id_given(tmp_path): - path = round_dir(tmp_path, 1, instance_id="task-1") - - assert path == tmp_path / "rounds_task-1" / "round_1" + # Config, memory and the training checkout survive. + assert (workdir / "task_config.yaml").read_text() == "repo: acme/widget" + assert (workdir / "memory" / "notes.md").read_text() == "learned" + assert (workdir / "repo" / "code.py").read_text() == "x = 1" + # Previous round output does not. + assert not (workdir / "rounds").exists() + assert any(p.name.startswith("workdir_backup_") for p in tmp_path.iterdir()) @pytest.mark.unit -def test_round_log_path_returns_round_log(tmp_path): - assert round_log_path(tmp_path, 2) == round_dir(tmp_path, 2) / "round.log" +def test_require_workdir_provides_memory_dir_even_when_previous_run_had_none(tmp_path): + workdir = tmp_path / "workdir" + workdir.mkdir() + (workdir / "task_config.yaml").write_text("repo: acme/widget") + require_workdir(workdir) -@pytest.mark.unit -def test_round_log_path_with_instance_id(tmp_path): - assert round_log_path(tmp_path, 2, instance_id="task-1") == round_dir( - tmp_path, 2, instance_id="task-1" - ) / "round.log" + assert memory_dir(workdir).is_dir() @pytest.mark.unit -def test_eval_dir_creates_directory_when_requested(tmp_path): - path = eval_dir(tmp_path, 1, "task-1", create=True) +def test_round_dir_is_idempotent(tmp_path): + first = round_dir(tmp_path, 1) + (first / "eval").mkdir() - assert path == tmp_path / "rounds_task-1" / "round_1" / "eval" - assert path.is_dir() + second = round_dir(tmp_path, 1) - -@pytest.mark.unit -def test_eval_dir_does_not_create_directory_by_default(tmp_path): - path = eval_dir(tmp_path, 1, "task-1") - - assert path == tmp_path / "rounds_task-1" / "round_1" / "eval" - assert not path.exists() + assert second == first + assert (second / "eval").is_dir(), "re-resolving a round must not wipe it" @pytest.mark.unit -def test_eval_result_path_returns_result_json(tmp_path): - assert eval_result_path(tmp_path, 1, "task-1") == eval_dir(tmp_path, 1, "task-1") / "result.json" +def test_take_memory_snapshot_copies_memory_into_the_round(tmp_path): + mem_dir = memory_dir(tmp_path) + mem_dir.mkdir(parents=True) + (mem_dir / "notes.md").write_text("round one") + take_memory_snapshot(mem_dir, 1) + (mem_dir / "notes.md").write_text("round two") -@pytest.mark.unit -def test_eval_log_path_returns_eval_log(tmp_path): - assert eval_log_path(tmp_path, 1, "task-1") == eval_dir(tmp_path, 1, "task-1") / "eval.log" + snapshot = round_dir(tmp_path, 1) / "starting_memory_snapshot" / "notes.md" + assert snapshot.read_text() == "round one" @pytest.mark.unit -def test_eval_patch_path_returns_repo_patch(tmp_path): - assert eval_patch_path(tmp_path, 1, "task-1") == eval_dir(tmp_path, 1, "task-1") / "repo.patch" +def test_take_memory_snapshot_requires_the_memory_dir_to_exist(tmp_path): + with pytest.raises(FileNotFoundError): + take_memory_snapshot(memory_dir(tmp_path), 1) From b0b968f7c8d60c8b109933bbcae99d9300d695ee Mon Sep 17 00:00:00 2001 From: bala Date: Tue, 8 Sep 2026 11:02:09 +0000 Subject: [PATCH 16/21] Increase the eval agent timeout --- .gitignore | 3 +++ src/microbots/auto_memory/eval/swebenchverified.py | 6 +++++- src/microbots/auto_memory/workdir/config.yaml | 3 --- 3 files changed, 8 insertions(+), 4 deletions(-) delete mode 100644 src/microbots/auto_memory/workdir/config.yaml diff --git a/.gitignore b/.gitignore index ca0ad18..d4fb80d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Any workdir +**/workdir*/ + # Microbots Project Specific .playwright-mcp/ diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index 0507ef7..0419e5d 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -358,7 +358,11 @@ def eval(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Bo folder_to_mount=repo_path, additional_tools=[MemoryTool(memory_dir=memory_dir)], ) - bot_result = bot.run(prompt) + bot_result = bot.run( + prompt, + max_iterations=40, + timeout_in_seconds=1800 + ) with open(log_path, "a") as f: f.write(f"Agent output:\n{bot_result.result}\n") diff --git a/src/microbots/auto_memory/workdir/config.yaml b/src/microbots/auto_memory/workdir/config.yaml deleted file mode 100644 index 03de1e5..0000000 --- a/src/microbots/auto_memory/workdir/config.yaml +++ /dev/null @@ -1,3 +0,0 @@ -instance_id_list: - - astropy__astropy-12907 - - astropy__astropy-13033 \ No newline at end of file From 8ace0db9793a9422ea51118a7193211ad5fc857b Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 9 Sep 2026 10:37:09 +0000 Subject: [PATCH 17/21] Update Azure CLI installation step to include package update --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a87c7a7..6c6fe93 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -121,7 +121,9 @@ jobs: - name: Reinstall Azure CLI (removed by disk cleanup) if: matrix.test-type == 'slow-browser' || matrix.test-type == 'slow-other' || matrix.test-type == 'ollama_local' - run: sudo apt-get install -y azure-cli + run: | + sudo apt-get update + sudo apt-get install -y azure-cli - name: Checkout code uses: actions/checkout@v4 From 955a40a5704b3f3b9929a81cca0b03cc2f4f8e27 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 9 Sep 2026 11:10:13 +0000 Subject: [PATCH 18/21] Enhance Azure CLI installation step with keyring setup and version check --- .github/workflows/test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6c6fe93..3537597 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -121,9 +121,17 @@ jobs: - name: Reinstall Azure CLI (removed by disk cleanup) if: matrix.test-type == 'slow-browser' || matrix.test-type == 'slow-other' || matrix.test-type == 'ollama_local' + shell: bash run: | + sudo install -d -m 0755 /etc/apt/keyrings + curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | + gpg --dearmor | sudo tee /etc/apt/keyrings/azure-cli.gpg > /dev/null + sudo chmod 0644 /etc/apt/keyrings/azure-cli.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/azure-cli.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | + sudo tee /etc/apt/sources.list.d/azure-cli.list > /dev/null sudo apt-get update sudo apt-get install -y azure-cli + az version - name: Checkout code uses: actions/checkout@v4 From b91b3330e34bec497388fb749c38b62762c77767 Mon Sep 17 00:00:00 2001 From: bala Date: Wed, 9 Sep 2026 15:49:16 +0000 Subject: [PATCH 19/21] Update logs to reach appropriate log files and fix the shared repo permission issue --- src/microbots/auto_memory/cli.py | 2 + .../auto_memory/eval/swebenchverified.py | 36 ++++---- src/microbots/auto_memory/orchestrator.py | 16 ++-- src/microbots/auto_memory/run_logging.py | 82 +++++++++++++++++++ src/microbots/auto_memory/workdir.py | 61 +++++++++++--- 5 files changed, 166 insertions(+), 31 deletions(-) create mode 100644 src/microbots/auto_memory/run_logging.py diff --git a/src/microbots/auto_memory/cli.py b/src/microbots/auto_memory/cli.py index e3f2294..a362a42 100644 --- a/src/microbots/auto_memory/cli.py +++ b/src/microbots/auto_memory/cli.py @@ -13,6 +13,7 @@ from pathlib import Path from microbots.auto_memory.orchestrator import run +from microbots.auto_memory.run_logging import configure_run_logging from microbots.auto_memory.task_registry import TASK_REGISTRY, discover_tasks from microbots.auto_memory.workdir import config_path, require_workdir, resolve_workdir @@ -76,6 +77,7 @@ def main(argv: list[str] | None = None) -> None: # the provided --config-file will take precedence. workdir = Path(args.workdir) if args.workdir else resolve_workdir() require_workdir(workdir) + configure_run_logging(workdir) if not args.config_file: config_file = config_path(workdir) diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index 0419e5d..fe892da 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -19,7 +19,9 @@ import yaml from microbots.auto_memory.evalTask import EvalOutcome, EvalTask +from microbots.auto_memory.run_logging import log_to_file from microbots.auto_memory.task_registry import register_task +from microbots.auto_memory.workdir import eval_log_dir from microbots.bot.ReadingBot import ReadingBot from microbots.bot.WritingBot import WritingBot from microbots.MicroBot import BotRunResult @@ -254,8 +256,10 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> BotRunResul subprocess.run( ["git", "add", "--intent-to-add", "."], cwd=repo_path, check=True ) + # Diffed against the base commit, so the patch is the same whether or + # not the agent committed its work. diff = subprocess.run( - ["git", "diff", "--binary"], + ["git", "diff", "--binary", self.instance.base_commit], cwd=repo_path, capture_output=True, text=True, @@ -460,8 +464,8 @@ def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: model : str The model to use, in the format ``/``. eval_dir : str - Directory this round's eval owns; holds the shared checkout - and one log file per instance. + Directory this round's eval owns; holds one checkout and one + log file per instance. Returns ------- @@ -470,22 +474,26 @@ def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: ``passed`` is true only when every one of them was. """ eval_path = Path(eval_dir) - eval_repo_path = eval_path / "eval_repo" - eval_log_dir = eval_path / "logs" + # One checkout per instance: the agent's container writes to it as + # root, leaving it unresettable for any instance that came after. + eval_repos_path = eval_path / "eval_repo" + log_dir = eval_log_dir(eval_path) results = [] for instance in self.dataset: - inst_log_path = eval_log_dir / f"{instance.instance_id}_log.txt" + inst_log_path = log_dir / f"{instance.instance_id}_log.txt" + inst_repo_path = eval_repos_path / instance.instance_id task = SweBenchVerifiedTask_one(instance) - res = task.eval(str(eval_repo_path), memory_dir, model, str(inst_log_path)) + with log_to_file(inst_log_path): + res = task.eval(str(inst_repo_path), memory_dir, model, str(inst_log_path)) - if not res.status: - logger.info(f"Evaluation failed for instance {instance.instance_id}: {res.error if res.error else 'Unknown error'}") - results.append(res) - else: - res = task.check(str(eval_repo_path), "", str(inst_log_path)) - results.append(res) + if not res.status: + logger.info(f"Evaluation failed for instance {instance.instance_id}: {res.error if res.error else 'Unknown error'}") + results.append(res) + else: + res = task.check(str(inst_repo_path), "", str(inst_log_path)) + results.append(res) score = 0 for result in results: @@ -497,7 +505,7 @@ def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: if score == 1: feedback = "All evaluations passed." else: - feedback = self._combine_result_feedback(results, model, str(eval_repo_path)) + feedback = self._combine_result_feedback(results, model, str(eval_repos_path)) # NOTE: Let's not teardown the repository as it will be useful for debugging diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index bf1a70e..46293eb 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -14,6 +14,7 @@ from pathlib import Path from microbots.auto_memory.evalTask import EvalOutcome, EvalTask +from microbots.auto_memory.run_logging import log_to_file from microbots.auto_memory.training.runner import run_training from microbots.auto_memory.workdir import ( RESULT_FILENAME, @@ -21,6 +22,7 @@ memory_dir, repo_dir, take_memory_snapshot, + training_log_path, ) logger = getLogger(__name__) @@ -150,7 +152,6 @@ def run_train_eval_loop( outcomes: list[EvalOutcome] = [] - #TODO: Logs need to be saved to appropriate log files for round_idx in range(1, max_rounds+1): logger.info( "run_train_eval_loop: round %d/%d starting", round_idx, max_rounds @@ -191,12 +192,13 @@ def run_train_eval_loop( ) try: - run_training( - repo_path=training_repo_path, - feedback=outcome.feedback, - memory_dir=str(mem_dir), - model=model, - ) + with log_to_file(training_log_path(workdir, round_idx)): + run_training( + repo_path=training_repo_path, + feedback=outcome.feedback, + memory_dir=str(mem_dir), + model=model, + ) except Exception: logger.exception( "run_train_eval_loop: round %d failed to build feedback/retrain; " diff --git a/src/microbots/auto_memory/run_logging.py b/src/microbots/auto_memory/run_logging.py new file mode 100644 index 0000000..37683b5 --- /dev/null +++ b/src/microbots/auto_memory/run_logging.py @@ -0,0 +1,82 @@ +"""Routes this package's existing log records into a run's workdir files. + +Three sinks, matching the workdir layout: the package's own records go +to ``workdir/log.txt``, while ``log_to_file`` temporarily captures the +agent/tool records emitted underneath a specific step (a training pass, +or one eval instance) into that step's own file. +""" + +import logging +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +from microbots.auto_memory.workdir import log_path + +PACKAGE_LOGGER_NAME = "microbots.auto_memory" +LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s: %(message)s" + + +def _file_handler(path: Path) -> logging.FileHandler: + """Build a formatted file handler appending to ``path``. + + Parameters + ---------- + path : Path + File to append records to. Parent directories are created. + + Returns + ------- + logging.FileHandler + The configured handler. + """ + path.parent.mkdir(parents=True, exist_ok=True) + handler = logging.FileHandler(path, encoding="utf-8") + handler.setFormatter(logging.Formatter(LOG_FORMAT)) + return handler + + +def configure_run_logging(workdir: Path) -> None: + """Send this package's own records to ``workdir/log.txt``. + + Propagation is disabled so these records never also land in the + per-step files that ``log_to_file`` attaches to the root logger. + Call once per run, after ``require_workdir`` has prepared the + workdir, otherwise the log file is archived out from under the + open handle. + + Parameters + ---------- + workdir : Path + The run's workdir. + """ + logger = logging.getLogger(PACKAGE_LOGGER_NAME) + logger.setLevel(logging.INFO) + logger.propagate = False + logger.addHandler(_file_handler(log_path(workdir))) + + +@contextmanager +def log_to_file(path: Path) -> Iterator[None]: + """Capture root-logger records into ``path`` for the duration of the block. + + Used to give a single training pass or eval instance its own file + without changing what any of the underlying agents/tools log. + + Parameters + ---------- + path : Path + File to append records to for the duration of the block. + """ + logger = logging.getLogger() + handler = _file_handler(path) + previous_level = logger.level + if not previous_level or previous_level > logging.INFO: + logger.setLevel(logging.INFO) + logger.addHandler(handler) + try: + yield + finally: + logger.removeHandler(handler) + logger.setLevel(previous_level) + handler.close() diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py index 3d35d63..047c700 100644 --- a/src/microbots/auto_memory/workdir.py +++ b/src/microbots/auto_memory/workdir.py @@ -22,8 +22,10 @@ """ STARTING_MEMORY_SNAPSHOT_DIR = "starting_memory_snapshot" ROUNDS_DIRNAME = "rounds" -ROUND_LOG_DIR= "logs" EVAL_DIRNAME = "eval" +EVAL_LOG_DIRNAME = "logs" +LOG_FILENAME = "log.txt" +TRAINING_LOG_FILENAME = "training_log.txt" RESULT_FILENAME = "result.json" """ @@ -31,12 +33,17 @@ workdir/ ├── task_config.yaml - ├── repo/ <-- Training checkout, reused across rounds - ├── memory/ <-- Mutated in place; the run's living memory + ├── log.txt <-- CLI and orchestrator logs + ├── repo/ <-- Training checkout, reused across rounds + ├── memory/ <-- Mutated in place; the run's living memory └── rounds/round_n/ - ├── logs/ <-- Training logs; eval logs live under eval/ - ├── eval/ <-- Managed by the eval task - └── starting_memory_snapshot/ <-- memory/ as it looked when the round began + ├── training_log.txt <-- Training agent logs for this round + ├── eval/ <-- Managed by the eval task + │ ├── result.json + │ ├── eval_repo/ + │ └── logs/ + │ └── _log.txt <-- One log per eval instance + └── starting_memory_snapshot/ <-- memory/ as it looked when the round began """ @@ -102,6 +109,22 @@ def config_path(workdir: Path) -> Path: return workdir / CONFIG_FILENAME +def log_path(workdir: Path) -> Path: + """Return the path to the run's top-level log file. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/log.txt``. + """ + return workdir / LOG_FILENAME + + def repo_dir(workdir: Path) -> Path: """Return the path to the training checkout shared across rounds. @@ -187,8 +210,8 @@ def round_dir( return path -def round_log_dir(workdir: Path, round_num: int) -> Path: - """Return the path to a round's training log directory. +def training_log_path(workdir: Path, round_num: int) -> Path: + """Return the path to a round's training log file. Parameters ---------- @@ -200,9 +223,9 @@ def round_log_dir(workdir: Path, round_num: int) -> Path: Returns ------- Path - ``workdir/rounds/round_{round_num}/logs``. + ``workdir/rounds/round_{round_num}/training_log.txt``. """ - return round_dir(workdir, round_num) / ROUND_LOG_DIR + return round_dir(workdir, round_num) / TRAINING_LOG_FILENAME def get_eval_dir( @@ -224,3 +247,21 @@ def get_eval_dir( path = round_dir(workdir, round_num) / EVAL_DIRNAME path.mkdir(parents=True, exist_ok=True) return path + + +def eval_log_dir(eval_dir: Path) -> Path: + """Return the directory holding one log file per eval instance. Creates it if missing. + + Parameters + ---------- + eval_dir : Path + The round's eval directory, as returned by ``get_eval_dir``. + + Returns + ------- + Path + ``/logs``. + """ + path = eval_dir / EVAL_LOG_DIRNAME + path.mkdir(parents=True, exist_ok=True) + return path From 087c67e06bdcf0a8ac84c98e1d6ef673eb7e6a40 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 16 Sep 2026 08:32:36 +0000 Subject: [PATCH 20/21] Update requirements and enhance MemoryTool for read-only mode --- requirements.txt | 2 +- .../auto_memory/eval/swebenchverified.py | 26 +- .../tools/tool_definitions/memory_tool.py | 247 +++++++++++++++++- 3 files changed, 260 insertions(+), 15 deletions(-) diff --git a/requirements.txt b/requirements.txt index bfb90ec..c408a65 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ attrs==25.3.0 bashlex==0.18 certifi==2025.8.3 charset-normalizer==3.4.3 -click==8.3.0 +click>=8.4.0,<9 coverage==7.11.3 distro==1.9.0 docker==7.1.0 diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index fe892da..76942a6 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -173,6 +173,13 @@ class SweBenchVerifiedTask_one: """ def __init__(self, instance: SweBenchInstance): + """Store the dataset instance this task evaluates against. + + Parameters + ---------- + instance : SweBenchInstance + The dataset instance this task evaluates against. + """ self.instance = instance def setup(self, repo_path: str) -> None: @@ -360,7 +367,7 @@ def eval(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Bo bot = WritingBot( model=model, folder_to_mount=repo_path, - additional_tools=[MemoryTool(memory_dir=memory_dir)], + additional_tools=[MemoryTool(memory_dir=memory_dir, read_only=True)], ) bot_result = bot.run( prompt, @@ -392,9 +399,22 @@ class SweBenchVerified(EvalTask): Every instance in the configured set is attempted with the same memory, and the round's score is the fraction that the harness marks resolved. + + Parameters + ---------- + config_file : Path + Path to the task's YAML config file, as consumed by + ``parse_config``. """ def __init__(self, config_file: Path) -> None: + """Load and validate the set of instances to evaluate against. + + Parameters + ---------- + config_file : Path + Path to the task's YAML config file. + """ # dataset must exist before parse_config populates it. self.dataset: list[SweBenchInstance] = [] self.parse_config(config_file=config_file) @@ -505,7 +525,9 @@ def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: if score == 1: feedback = "All evaluations passed." else: - feedback = self._combine_result_feedback(results, model, str(eval_repos_path)) + combine_log_path = log_dir / "combine_result_feedback_log.txt" + with log_to_file(combine_log_path): + feedback = self._combine_result_feedback(results, model, str(eval_repos_path)) # NOTE: Let's not teardown the repository as it will be useful for debugging diff --git a/src/microbots/tools/tool_definitions/memory_tool.py b/src/microbots/tools/tool_definitions/memory_tool.py index 0dd5ec2..ac737e7 100644 --- a/src/microbots/tools/tool_definitions/memory_tool.py +++ b/src/microbots/tools/tool_definitions/memory_tool.py @@ -1,3 +1,10 @@ +"""File-backed ``memory`` tool exposed to agents via the text command loop. + +Provides ``MemoryTool``, an ``ExternalTool`` that lets an LLM view, create, +and edit files under a host ``memory_dir`` using ``/memories/...`` paths, +optionally restricted to read-only access. +""" + import argparse import logging import os @@ -18,24 +25,50 @@ class _NoExitArgumentParser(argparse.ArgumentParser): """ArgumentParser that raises ``ValueError`` instead of calling ``sys.exit``.""" def error(self, message: str) -> None: # type: ignore[override] + """Raise ``ValueError`` instead of printing usage and exiting. + + Parameters + ---------- + message : str + The error message argparse would otherwise print. + + Raises + ------ + ValueError + Always raised, carrying ``message``. + """ raise ValueError(message) -INSTRUCTIONS_TO_LLM = """ +# Pieces shared by both the read/write and read-only instructions, so the +# two variants can't drift apart on what they say about `view` and paths. +_VIEW_PROTOCOL_STEP = ( + "ALWAYS run `memory view /memories` BEFORE doing anything else to check " + "for earlier progress." +) + +_VIEW_COMMANDS = """View a file or list a directory: + memory view + memory view --start --end """ + +_VIEW_EXAMPLES = """ memory view /memories + memory view /memories/progress.md --start 1 --end 10""" + +_COMMON_NOTES = """- Paths must start with /memories/. +- In memory view, use --end -1 to read through the end of the file.""" + +INSTRUCTIONS_TO_LLM = f""" Use this tool to persist information to files across steps. All paths must be under /memories/. MEMORY PROTOCOL: -1. ALWAYS run `memory view /memories` BEFORE doing anything else to check for - earlier progress. +1. {_VIEW_PROTOCOL_STEP} 2. Record status, findings and intermediate results as you go. 3. Before completing a task, save your final results to memory. 4. Keep the memory folder organised — rename or delete stale files. ## Commands -View a file or list a directory: - memory view - memory view --start --end +{_VIEW_COMMANDS} Create a file: memory create @@ -57,19 +90,38 @@ def error(self, message: str) -> None: # type: ignore[override] ## Examples - memory view /memories +{_VIEW_EXAMPLES} memory create /memories/progress.md "## Progress\\n- Found bug in src/foo.py line 42" memory str_replace /memories/progress.md --old "line 42" --new "line 45" memory insert /memories/progress.md --line 0 --text "# Task Notes" - memory view /memories/progress.md --start 1 --end 10 memory delete /memories/old_notes.md memory rename /memories/draft.md /memories/final.md ## Notes -- Paths must start with /memories/. +{_COMMON_NOTES} - memory create overwrites if the file already exists. - memory str_replace requires the old text to appear exactly once. -- In memory view, use --end -1 to read through the end of the file. +""" + +READ_ONLY_INSTRUCTIONS_TO_LLM = f""" +Use this tool to read previously recorded memory files. +All paths must be under /memories/. +Only `memory view` is available — this memory store is read-only, so +create/str_replace/insert/delete/rename/clear will all be rejected. + +MEMORY PROTOCOL: +1. {_VIEW_PROTOCOL_STEP} + +## Commands + +{_VIEW_COMMANDS} + +## Examples + +{_VIEW_EXAMPLES} + +## Notes +{_COMMON_NOTES} """ @@ -94,15 +146,46 @@ class MemoryTool(ExternalTool): ) usage_instructions_to_llm: str = Field(default=INSTRUCTIONS_TO_LLM) memory_dir: Optional[str] = Field(default=None) + read_only: bool = Field(default=False) + """ + When true, only ``view`` is permitted; every mutating subcommand + (create/str_replace/insert/delete/rename/clear) is rejected before it + touches the filesystem. Used to give an agent read access to existing + memory without letting it change what future runs see. + """ def __post_init__(self): + """Resolve ``memory_dir``, create it, and finalize read-only defaults. + + Falls back to ``~/.microbots/memory`` when ``memory_dir`` is + unset. When ``read_only`` is true, swaps the default + description/instructions for their read-only equivalents + (leaving any explicitly supplied override untouched), and builds + the command parser. + """ base = Path(self.memory_dir) if self.memory_dir else Path.home() / ".microbots" / "memory" self._memory_dir = base self._memory_dir.mkdir(parents=True, exist_ok=True) + + # Only swap the default copy; an explicitly supplied description/ + # instructions is left alone even in read-only mode. + if self.read_only: + if self.usage_instructions_to_llm == INSTRUCTIONS_TO_LLM: + self.usage_instructions_to_llm = READ_ONLY_INSTRUCTIONS_TO_LLM + if self.description == "File-backed memory store — view, create, edit, delete files under /memories/.": + self.description = "Read-only file-backed memory store — view files under /memories/." self._parser = self._build_parser() def _build_parser(self) -> _NoExitArgumentParser: - """Build the argparse parser with subparsers for each memory subcommand.""" + """Build the argparse parser with subparsers for each memory subcommand. + + Returns + ------- + _NoExitArgumentParser + Parser configured with one subparser per memory subcommand + (``view``, ``create``, ``str_replace``, ``insert``, + ``delete``, ``rename``, ``clear``). + """ parser = _NoExitArgumentParser(prog="memory", add_help=False) subs = parser.add_subparsers(dest="subcommand") @@ -141,7 +224,25 @@ def _build_parser(self) -> _NoExitArgumentParser: # ---------------------------------------------------------------------- # def _resolve(self, path: str) -> Path: - """Resolve a /memories/… path to an absolute host path.""" + """Resolve a /memories/… path to an absolute host path. + + Parameters + ---------- + path : str + An LLM-supplied path, expected to start with ``/memories``. + + Returns + ------- + Path + The absolute host path under ``memory_dir`` that ``path`` + refers to. + + Raises + ------ + ValueError + If ``path`` doesn't start with ``/memories`` or attempts to + traverse outside ``memory_dir``. + """ if not path.startswith("/"): raise ValueError( f"Invalid memory path: {path!r}. Paths must start with /memories/." @@ -175,10 +276,39 @@ def _resolve(self, path: str) -> Path: # ---------------------------------------------------------------------- # def is_invoked(self, command: str) -> bool: + """Return whether ``command`` is a ``memory`` invocation. + + Parameters + ---------- + command : str + The raw command text the agent issued. + + Returns + ------- + bool + True if ``command`` is (or starts with) ``memory``. + """ cmd = command.strip() return cmd == "memory" or cmd.startswith("memory ") def invoke(self, command: str, parent_bot) -> CmdReturn: + """Parse and dispatch a ``memory`` command to its subcommand handler. + + Parameters + ---------- + command : str + The full ``memory ...`` command text. + parent_bot : MicroBot + The bot invoking this tool. Unused; accepted for interface + compatibility with ``ToolAbstract``. + + Returns + ------- + CmdReturn + The subcommand's result, or an error result if parsing + failed, the subcommand is unknown, read-only mode blocks + it, or the handler raised. + """ try: tokens = shlex.split(command) except ValueError as exc: @@ -192,6 +322,13 @@ def invoke(self, command: str, parent_bot) -> CmdReturn: if args.subcommand is None: return CmdReturn(stdout="", stderr="Usage: memory ...", return_code=1) + if self.read_only and args.subcommand != "view": + return CmdReturn( + stdout="", + stderr=f"Memory is read-only: '{args.subcommand}' is not permitted.", + return_code=1, + ) + dispatch = { "view": self._view, "create": self._create, @@ -214,6 +351,19 @@ def invoke(self, command: str, parent_bot) -> CmdReturn: # ---------------------------------------------------------------------- # def _view(self, args: argparse.Namespace) -> CmdReturn: + """Handle ``memory view`` — list a directory or print a file's lines. + + Parameters + ---------- + args : argparse.Namespace + Parsed ``view`` arguments: ``path``, ``start``, ``end``. + + Returns + ------- + CmdReturn + The directory listing or numbered file lines, or an error + result if ``path`` doesn't exist. + """ resolved = self._resolve(args.path) if not resolved.exists(): return CmdReturn(stdout="", stderr=f"Path not found: {args.path!r}", return_code=1) @@ -239,6 +389,19 @@ def _view(self, args: argparse.Namespace) -> CmdReturn: return CmdReturn(stdout=numbered, stderr="", return_code=0) def _create(self, args: argparse.Namespace) -> CmdReturn: + """Handle ``memory create`` — write (or overwrite) a file. + + Parameters + ---------- + args : argparse.Namespace + Parsed ``create`` arguments: ``path``, ``content``. + + Returns + ------- + CmdReturn + Confirmation of the write, or an error result if no + content was supplied. + """ if not args.content: return CmdReturn(stdout="", stderr="Usage: memory create ", return_code=1) content = " ".join(args.content) @@ -249,6 +412,19 @@ def _create(self, args: argparse.Namespace) -> CmdReturn: return CmdReturn(stdout=f"File created: {args.path}", stderr="", return_code=0) def _str_replace(self, args: argparse.Namespace) -> CmdReturn: + """Handle ``memory str_replace`` — replace a unique substring in a file. + + Parameters + ---------- + args : argparse.Namespace + Parsed ``str_replace`` arguments: ``path``, ``old``, ``new``. + + Returns + ------- + CmdReturn + Confirmation of the edit, or an error result if the file + doesn't exist or ``old`` isn't found exactly once. + """ resolved = self._resolve(args.path) if not resolved.is_file(): return CmdReturn(stdout="", stderr=f"File not found: {args.path!r}", return_code=1) @@ -262,6 +438,19 @@ def _str_replace(self, args: argparse.Namespace) -> CmdReturn: return CmdReturn(stdout=f"File {args.path} has been edited.", stderr="", return_code=0) def _insert(self, args: argparse.Namespace) -> CmdReturn: + """Handle ``memory insert`` — insert a line at a given position. + + Parameters + ---------- + args : argparse.Namespace + Parsed ``insert`` arguments: ``path``, ``line``, ``text``. + + Returns + ------- + CmdReturn + Confirmation of the insert, or an error result if the file + doesn't exist or ``line`` is out of range. + """ resolved = self._resolve(args.path) if not resolved.is_file(): return CmdReturn(stdout="", stderr=f"File not found: {args.path!r}", return_code=1) @@ -273,6 +462,19 @@ def _insert(self, args: argparse.Namespace) -> CmdReturn: return CmdReturn(stdout=f"Text inserted at line {args.line} in {args.path}.", stderr="", return_code=0) def _delete(self, args: argparse.Namespace) -> CmdReturn: + """Handle ``memory delete`` — remove a file or directory. + + Parameters + ---------- + args : argparse.Namespace + Parsed ``delete`` arguments: ``path``. + + Returns + ------- + CmdReturn + Confirmation of the deletion, or an error result if + ``path`` is the memory root or doesn't exist. + """ resolved = self._resolve(args.path) if resolved == self._memory_dir.resolve(): return CmdReturn(stdout="", stderr="Cannot delete the /memories root directory", return_code=1) @@ -287,6 +489,20 @@ def _delete(self, args: argparse.Namespace) -> CmdReturn: return CmdReturn(stdout="", stderr=f"Path not found: {args.path!r}", return_code=1) def _rename(self, args: argparse.Namespace) -> CmdReturn: + """Handle ``memory rename`` — move/rename a file or directory. + + Parameters + ---------- + args : argparse.Namespace + Parsed ``rename`` arguments: ``old_path``, ``new_path``. + + Returns + ------- + CmdReturn + Confirmation of the rename, or an error result if either + path is the memory root, the source is missing, or the + destination already exists. + """ old_resolved = self._resolve(args.old_path) new_resolved = self._resolve(args.new_path) memory_root = self._memory_dir.resolve() @@ -304,6 +520,13 @@ def _rename(self, args: argparse.Namespace) -> CmdReturn: return CmdReturn(stdout=f"Renamed {args.old_path} to {args.new_path}.", stderr="", return_code=0) def _clear(self) -> CmdReturn: + """Handle ``memory clear`` — delete and recreate the memory root. + + Returns + ------- + CmdReturn + Confirmation that memory was cleared. + """ if self._memory_dir.exists(): shutil.rmtree(self._memory_dir) self._memory_dir.mkdir(parents=True, exist_ok=True) From 432e1033025a29d5a0c481cbcc900b3945524ed2 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 16 Sep 2026 10:55:15 +0000 Subject: [PATCH 21/21] Refactor eval method signatures to include training_repo_dir parameter for enhanced evaluation context and improve prompt --- src/microbots/MicroBot.py | 132 +++++++++++++++--- src/microbots/auto_memory/architecture.md | 2 +- .../auto_memory/eval/swebenchverified.py | 50 +++++-- src/microbots/auto_memory/evalTask.py | 21 ++- src/microbots/auto_memory/orchestrator.py | 2 +- src/microbots/llm/llm.py | 61 +++++++- .../auto_memory/eval/test_swebenchverified.py | 6 +- test/auto_memory/test_task_registry.py | 2 +- 8 files changed, 234 insertions(+), 42 deletions(-) diff --git a/src/microbots/MicroBot.py b/src/microbots/MicroBot.py index 7402288..2c347e9 100644 --- a/src/microbots/MicroBot.py +++ b/src/microbots/MicroBot.py @@ -1,3 +1,4 @@ +"""Core MicroBot agent class and supporting types.""" from collections.abc import Iterable import json import os @@ -53,6 +54,8 @@ class BotType(StrEnum): + """Enumeration of the supported bot types.""" + READING_BOT = "READING_BOT" WRITING_BOT = "WRITING_BOT" BROWSING_BOT = "BROWSING_BOT" @@ -62,6 +65,19 @@ class BotType(StrEnum): @dataclass class BotRunResult: + """ + Result of a MicroBot run. + + Attributes + ---------- + status : bool + True if the bot completed the task successfully, False otherwise. + result : str | None + The final result/output produced by the bot, or None if unavailable. + error : Optional[str] + An error message if the run failed, or None if it succeeded. + """ + status: bool result: str | None error: Optional[str] @@ -74,7 +90,7 @@ class MicroBot: MicroBot class is the core class representing the autonomous agent. Other bots are extensions of this class. If you want to create a custom bot, you can directly use this class or extend it into your own bot class. - Attributes + Parameters ---------- model : str The model to use for the bot, in the format /. @@ -94,6 +110,10 @@ class MicroBot: can be mounted during the run() method. Refer to `Mount` class regarding the directory structure and permission details. Defaults to None. + token_provider : Optional[any] + A callable that returns a bearer token for Azure AD authentication. + If not provided, it may be auto-created from environment variables. + Defaults to None. """ def __init__( @@ -111,19 +131,19 @@ def __init__( Parameters ---------- - model :str + model : str The model to use for the bot, in the format /. - bot_type :BotType + bot_type : BotType The type of bot being created. It's unused. Will be removed soon. - system_prompt :Optional[str] + system_prompt : Optional[str] The system prompt to guide the bot's behavior. Defaults to None. - environment :Optional[any] + environment : Optional[any] The execution environment for the bot. If not provided, a default LocalDockerEnvironment will be created. - additional_tools :Optional[list[ToolAbstract]] + additional_tools : Optional[list[ToolAbstract]] A list of additional tools to install in the bot's environment. Defaults to None (treated as an empty list). - folder_to_mount :Optional[Mount] + folder_to_mount : Optional[Mount] A folder to mount into the bot's environment. The bot will be given access to this folder based on the specified permissions. This will be the main code folder where the bot will work. Additional folders @@ -132,6 +152,10 @@ def __init__( to None. Note: Supports only mount type MountType.MOUNT for now. + token_provider : Optional[any] + A callable that returns a bearer token for Azure AD authentication. + If not provided, it may be auto-created from environment variables. + Defaults to None. """ self.folder_to_mount = folder_to_mount @@ -203,6 +227,28 @@ def run( max_iterations: int = 20, timeout_in_seconds: int = 200 ) -> BotRunResult: + """ + Run the bot on the given task until completion, timeout, or max iterations. + + Parameters + ---------- + task : str + The task description to give to the bot. + additional_mounts : Optional[list[Mount]] + Additional folders to mount into the bot's environment before + running. Defaults to None. + max_iterations : int + The maximum number of LLM interaction iterations allowed before + aborting the task. Defaults to 20. + timeout_in_seconds : int + The maximum wall-clock time in seconds allowed for the task. + Defaults to 200. + + Returns + ------- + BotRunResult + The outcome of the run, including status, result, and error. + """ if max_iterations <= 0: raise ValueError("max_iterations must be greater than 0") @@ -311,9 +357,18 @@ def run( f" 💭 LLM final thoughts: {llm_response.thoughts}", ) logger.info("🔚 TASK COMPLETED : %s...", task[0:15]) - return BotRunResult(status=True, result=llm_response.thoughts, error=None) + return BotRunResult(status=True, result=llm_response.result or llm_response.thoughts, error=None) def _mount_additional(self, mount: Mount): + """ + Copy an additional folder into the bot's running environment. + + Parameters + ---------- + mount : Mount + The additional mount to copy into the environment. Only + MountType.COPY mounts are supported. + """ if mount.mount_type != MountType.COPY: logger.error( "%s Only COPY mount type is supported for additional mounts for now", @@ -334,6 +389,14 @@ def _mount_additional(self, mount: Mount): # TODO : pass the sandbox path def _create_environment(self, folder_to_mount: Optional[Mount]): + """ + Create the LocalDockerEnvironment for the bot on a free host port. + + Parameters + ---------- + folder_to_mount : Optional[Mount] + The folder to mount into the created environment. + """ free_port = get_free_port() self.environment = LocalDockerEnvironment( @@ -342,6 +405,7 @@ def _create_environment(self, folder_to_mount: Optional[Mount]): ) def _create_llm(self): + """Create the LLM client for the configured model provider.""" # Append tool usage instructions to system prompt system_prompt_with_tools = self.system_prompt if self.system_prompt else "" if self.additional_tools: @@ -370,6 +434,14 @@ def _create_llm(self): # No Else case required as model provider is already validated using _validate_model_and_provider def _validate_model_and_provider(self, model): + """ + Validate that the model string is well-formed and its provider is supported. + + Parameters + ---------- + model : str + The model string in the format /. + """ # Ensure it has only only slash if model.count("/") != 1: raise ValueError("Model should be in the format /") @@ -378,6 +450,14 @@ def _validate_model_and_provider(self, model): raise ValueError(f"Unsupported model provider: {provider}") def _validate_folder_to_mount(self, folder_to_mount: Mount): + """ + Validate that the folder to mount uses a supported mount type. + + Parameters + ---------- + folder_to_mount : Mount + The mount to validate. Only MountType.MOUNT is supported. + """ if folder_to_mount.mount_type != MountType.MOUNT: logger.error( "%s Only MOUNT mount type is supported for folder_to_mount", @@ -388,13 +468,18 @@ def _validate_folder_to_mount(self, folder_to_mount: Mount): ) def _get_dangerous_command_explanation(self, command: str) -> Optional[str]: - """Provides detailed explanation for why a command is dangerous and suggests alternatives. + """ + Provide a detailed explanation for why a command is dangerous and suggest alternatives. - Args: - command: The shell command to analyze + Parameters + ---------- + command : str + The shell command to analyze. - Returns: - str: Explanation with reason and alternative, or None if command is safe + Returns + ------- + Optional[str] + Explanation with reason and alternative, or None if command is safe. """ # Handle invalid commands (empty, None, or non-string) if not command or not isinstance(command, str): @@ -441,19 +526,24 @@ def _get_dangerous_command_explanation(self, command: str) -> Optional[str]: return None def _is_safe_command(self, command: str) -> tuple[bool, Optional[str]]: - """Validates if a command is safe to execute. + """ + Validate whether a command is safe to execute. A command is considered safe if it: - Is not a recursive command (ls -R, rm -rf, tree, find without -maxdepth) - Does not risk generating excessive output or destructive actions - Args: - command: The shell command to validate - - Returns: - tuple[bool, Optional[str]]: A tuple of (is_safe, explanation) where: - - is_safe: True if command is safe to execute, False otherwise - - explanation: Detailed explanation if dangerous, None if safe + Parameters + ---------- + command : str + The shell command to validate. + + Returns + ------- + tuple[bool, Optional[str]] + A tuple of (is_safe, explanation) where is_safe is True if the + command is safe to execute, and explanation is a detailed + explanation if dangerous, or None if safe. """ explanation = self._get_dangerous_command_explanation(command) is_safe = explanation is None diff --git a/src/microbots/auto_memory/architecture.md b/src/microbots/auto_memory/architecture.md index 12c2418..117ce58 100644 --- a/src/microbots/auto_memory/architecture.md +++ b/src/microbots/auto_memory/architecture.md @@ -172,7 +172,7 @@ class MyTask(EvalTask): def parse_config(self, config_file: Path) -> None: ... # load your settings; set self._repo_url or override repo_url() - def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: + def eval(self, memory_dir: str, model: str, eval_dir: str, training_repo_dir: str) -> EvalOutcome: ... # run every unit of work, return one combined outcome ``` diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index 76942a6..7974f46 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -474,7 +474,7 @@ def parse_config(self, config_file: Path) -> None: if len(self.dataset) == 0: raise ValueError("No instances loaded for evaluation.") - def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: + def eval(self, memory_dir: str, model: str, eval_dir: str, training_repo_dir: str) -> EvalOutcome: """Attempt every configured instance and combine the results. Parameters @@ -486,6 +486,9 @@ def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: eval_dir : str Directory this round's eval owns; holds one checkout and one log file per instance. + training_repo_dir : str + Absolute path to the persistent training checkout, mounted + for the bot that combines instance results into feedback. Returns ------- @@ -527,7 +530,7 @@ def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: else: combine_log_path = log_dir / "combine_result_feedback_log.txt" with log_to_file(combine_log_path): - feedback = self._combine_result_feedback(results, model, str(eval_repos_path)) + feedback = self._combine_result_feedback(results, model, training_repo_dir) # NOTE: Let's not teardown the repository as it will be useful for debugging @@ -537,7 +540,7 @@ def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: feedback = feedback ) - def _combine_result_feedback(self, results: list[BotRunResult], model: str, eval_repo: str) -> str: + def _combine_result_feedback(self, results: list[BotRunResult], model: str, training_repo_dir: str) -> str: """Summarize every instance's result into one feedback string. Parameters @@ -546,8 +549,9 @@ def _combine_result_feedback(self, results: list[BotRunResult], model: str, eval One result per attempted instance. model : str The model to use, in the format ``/``. - eval_repo : str - Path to the evaluation checkout, mounted for the bot. + training_repo_dir : str + Absolute path to the persistent training checkout, mounted + for the bot. Returns ------- @@ -566,13 +570,39 @@ def _combine_result_feedback(self, results: list[BotRunResult], model: str, eval try: bot = ReadingBot( model = model, - folder_to_mount=eval_repo + folder_to_mount=training_repo_dir ) task = f""" - Combine the results of the eval runs into single feedback. - This feedback will be given to the next iteration. - You just combine the results with minimal efforts. - Avoid referring to code whenever possible. + You are combining results from {len(results)} SWE-bench evaluation + runs into ONE feedback report for the next training iteration. The + training agent will read your report to decide what to add or fix + in its memory notes. + + For each result below, note whether it passed or failed. For each + failure, briefly identify the underlying cause (e.g. wrong + file/line targeted, incorrect patch logic, response format error, + timeout) rather than only quoting the raw error. You may open + files under the mounted repo if you need to confirm a root + cause, but do not turn this into a debugging session. + Do not refer to any specific instance or test case by name/ID — + describe causes and guidance in general terms only. + + Then write a report with: + 1. A one-line summary: how many passed vs failed. + 2. Grouped failure patterns: if multiple failures share the same + root cause, describe that cause once rather than repeating + yourself. + 3. Concrete, actionable guidance for the training agent — say + what to change in the memory notes to avoid each failure + pattern next time. Be specific and imperative + (e.g. "Record that config paths must be normalized before + comparison", not "there was a path issue"). + 4. Skip anything about passed cases beyond the summary count; + don't restate their feedback. + + Keep the report tight and skimmable — short paragraphs or bullet + points, no code dumps. Put the final report in the `result` + field once you set task_done=true. {serialized_str} """ diff --git a/src/microbots/auto_memory/evalTask.py b/src/microbots/auto_memory/evalTask.py index 50a791c..a10b549 100644 --- a/src/microbots/auto_memory/evalTask.py +++ b/src/microbots/auto_memory/evalTask.py @@ -37,11 +37,23 @@ class EvalTask(ABC): ``repo_url`` have working defaults driven by the config file's ``repo`` key, and may be overridden by tasks that derive the repo some other way (see ``SweBenchVerified``). + + Parameters + ---------- + config_file : Path + Path to the task's config file, parsed during initialization. """ _repo_url: str | None = None def __init__(self, config_file: Path) -> None: + """Parse the config file and record the training repo URL. + + Parameters + ---------- + config_file : Path + Path to the task's config file. + """ # NOTE: Don't call this from child class unless you need to reuse # the parse_config logic from here. super().__init__() @@ -82,7 +94,7 @@ def parse_config(self, config_file: Path) -> None: self._repo_url = config.get("repo") @abstractmethod - def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: + def eval(self, memory_dir: str, model: str, eval_dir: str, training_repo_dir: str) -> EvalOutcome: """Required. Run one full evaluation and return its outcome. Parameters @@ -92,13 +104,14 @@ def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: ``MemoryTool``. model : str The model to use, in the format ``/``. - eval_dir: str + eval_dir : str Directory this round's eval owns. The task decides what goes in it (cloned repo, logs, and so on). + training_repo_dir : str + Absolute path to the persistent training checkout. Returns ------- EvalOutcome - Whether the round passed, its score, and the feedback to - retrain on. + Whether the round passed, its score, and the feedback to use for retraining. """ diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index 46293eb..748e4dd 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -160,7 +160,7 @@ def run_train_eval_loop( eval_dir = get_eval_dir(workdir, round_idx) take_memory_snapshot(mem_dir, round_idx) try: - outcome = task.eval(str(mem_dir), model, str(eval_dir)) + outcome = task.eval(str(mem_dir), model, str(eval_dir), training_repo_path) outcomes.append(outcome) except Exception as e: logger.warning( diff --git a/src/microbots/llm/llm.py b/src/microbots/llm/llm.py index 2800790..a29c9df 100644 --- a/src/microbots/llm/llm.py +++ b/src/microbots/llm/llm.py @@ -1,3 +1,4 @@ +"""LLM interface and response parsing/validation for MicroBot.""" from dataclasses import dataclass from abc import ABC, abstractmethod import json @@ -16,20 +17,75 @@ @dataclass class LLMAskResponse: + """ + Parsed response from an LLM turn. + + Attributes + ---------- + task_done : bool + Whether the LLM considers the task complete. + thoughts : str + The LLM's reasoning behind its decision. + command : str + The command the LLM wants executed next. + result : str + Optional final result populated by task-specific prompts + (e.g. ReadingBot) when task_done is True. + """ + task_done: bool = False thoughts: str = "" command: str = "" + result: str = "" class LLMInterface(ABC): + """Abstract interface for an LLM client used by MicroBot.""" + @abstractmethod def ask(self, message: str) -> LLMAskResponse: + """ + Send a message to the LLM and return its parsed response. + + Parameters + ---------- + message : str + The message/prompt to send to the LLM. + + Returns + ------- + LLMAskResponse + The parsed LLM response. + """ pass @abstractmethod def clear_history(self) -> bool: + """ + Clear the LLM's conversation history. + + Returns + ------- + bool + True if the history was cleared successfully. + """ pass def _validate_llm_response(self, response: str) -> tuple[bool, LLMAskResponse]: + """ + Validate and parse a raw LLM response string into an LLMAskResponse. + + Parameters + ---------- + response : str + The raw response text returned by the LLM. + + Returns + ------- + tuple[bool, LLMAskResponse] + A tuple of (is_valid, parsed_response). is_valid is False and + parsed_response is None when the response could not be parsed + or failed validation. + """ if self.retries >= self.max_retries: logger.error("Maximum retries reached for LLM response validation.") @@ -43,7 +99,9 @@ def _validate_llm_response(self, response: str) -> tuple[bool, LLMAskResponse]: self.messages.append({"role": "user", "content": "LLM_RES_ERROR: Please respond in the correct JSON format.\n" + llm_output_format_str}) return False, None - if all(key in response_dict for key in LLMAskResponse.__annotations__.keys()): + # "result" is optional, so it's excluded from this required-keys check. + required_keys = ("task_done", "thoughts", "command") + if all(key in response_dict for key in required_keys): logger.info("The llm response is %s ", response_dict) if response_dict.get("task_done") not in [True, False]: @@ -77,6 +135,7 @@ def _validate_llm_response(self, response: str) -> tuple[bool, LLMAskResponse]: task_done=response_dict["task_done"], command=response_dict["command"], thoughts=response_dict.get("thoughts"), + result=response_dict.get("result", ""), ) return True, llm_response else: diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py index 71e565d..d07f569 100644 --- a/test/auto_memory/eval/test_swebenchverified.py +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -156,7 +156,7 @@ def test_eval_scores_the_fraction_of_resolved_instances(tmp_path): reading_bot.return_value.run.return_value = BotRunResult( status=True, result="one instance still fails", error=None ) - outcome = task.eval(str(tmp_path / "memory"), "azure-openai/gpt-4o", str(tmp_path / "eval")) + outcome = task.eval(str(tmp_path / "memory"), "azure-openai/gpt-4o", str(tmp_path / "eval"), str(tmp_path / "repo")) assert outcome.score == 0.5 assert not outcome.passed @@ -175,7 +175,7 @@ def test_eval_passes_only_when_every_instance_resolves(tmp_path): writing_bot.return_value.run.return_value = BotRunResult( status=True, result="patched", error=None ) - outcome = task.eval(str(tmp_path / "memory"), "azure-openai/gpt-4o", str(tmp_path / "eval")) + outcome = task.eval(str(tmp_path / "memory"), "azure-openai/gpt-4o", str(tmp_path / "eval"), str(tmp_path / "repo")) assert outcome.passed assert outcome.score == 1 @@ -197,7 +197,7 @@ def test_eval_skips_the_harness_when_the_agent_itself_failed(tmp_path): reading_bot.return_value.run.return_value = BotRunResult( status=True, result="the agent never produced a patch", error=None ) - outcome = task.eval(str(tmp_path / "memory"), "azure-openai/gpt-4o", str(tmp_path / "eval")) + outcome = task.eval(str(tmp_path / "memory"), "azure-openai/gpt-4o", str(tmp_path / "eval"), str(tmp_path / "repo")) check.assert_not_called() assert outcome.score == 0 diff --git a/test/auto_memory/test_task_registry.py b/test/auto_memory/test_task_registry.py index f761331..0389e08 100644 --- a/test/auto_memory/test_task_registry.py +++ b/test/auto_memory/test_task_registry.py @@ -18,7 +18,7 @@ class _StubTask(EvalTask): def parse_config(self, config_file: Path) -> None: self._repo_url = "https://github.com/acme/widget.git" - def eval(self, memory_dir: str, model: str, eval_dir: str) -> EvalOutcome: + def eval(self, memory_dir: str, model: str, eval_dir: str, training_repo_dir: str) -> EvalOutcome: return EvalOutcome(passed=True, score=1.0, feedback="ok")