diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e898772c..35375977 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -121,7 +121,17 @@ jobs: - name: Reinstall Azure CLI (removed by disk cleanup) if: matrix.test-type == 'slow-browser' || matrix.test-type == 'slow-other' || matrix.test-type == 'ollama_local' - run: sudo apt-get install -y azure-cli + shell: bash + run: | + sudo install -d -m 0755 /etc/apt/keyrings + curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | + gpg --dearmor | sudo tee /etc/apt/keyrings/azure-cli.gpg > /dev/null + sudo chmod 0644 /etc/apt/keyrings/azure-cli.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/azure-cli.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | + sudo tee /etc/apt/sources.list.d/azure-cli.list > /dev/null + sudo apt-get update + sudo apt-get install -y azure-cli + az version - name: Checkout code uses: actions/checkout@v4 @@ -165,7 +175,7 @@ jobs: - name: Install package in development mode run: | - pip install -e . + pip install -e ".[training]" - name: Install GitHub Copilot SDK dependencies for GHCP tests if: matrix.test-type == 'ghcp' diff --git a/.gitignore b/.gitignore index ca0ad18e..d4fb80d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Any workdir +**/workdir*/ + # Microbots Project Specific .playwright-mcp/ diff --git a/pyproject.toml b/pyproject.toml index 3917589a..a66a1384 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ requires-python = ">=3.11" ghcp = ["github-copilot-sdk==0.3.0"] azure_ad = ["azure-identity>=1.15.0"] dev = ["pre-commit>=3.7", "numpydoc>=1.8"] +training = ["datasets==4.5.0", "swebench==4.1.0"] [tool.setuptools.dynamic] dependencies = { file = ["requirements.txt"] } diff --git a/requirements.txt b/requirements.txt index 4bf03b51..c408a65e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,38 +8,28 @@ attrs==25.3.0 bashlex==0.18 certifi==2025.8.3 charset-normalizer==3.4.3 -click==8.3.0 +click>=8.4.0,<9 coverage==7.11.3 -datasets==4.5.0 -dill==0.4.0 distro==1.9.0 docker==7.1.0 docstring_parser==0.17.0 fastapi==0.116.1 -filelock==3.20.3 frozenlist==1.7.0 -fsspec==2025.10.0 h11==0.16.0 -hf-xet==1.2.0 httpcore==1.0.9 httpx==0.28.1 -huggingface_hub==1.3.2 idna==3.10 iniconfig==2.1.0 jiter==0.11.0 markdown-it-py==4.0.0 mdurl==0.1.2 multidict==6.6.4 -multiprocess==0.70.18 -numpy==1.26.4 openai==1.107.3 packaging==25.0 -pandas==3.0.0 pexpect==4.9.0 pluggy==1.6.0 propcache==0.3.2 ptyprocess==0.7.0 -pyarrow==23.0.0 pydantic==2.11.9 pydantic_core==2.33.2 Pygments==2.19.2 @@ -62,5 +52,4 @@ typing-inspection==0.4.1 typing_extensions==4.15.0 urllib3==2.5.0 uvicorn==0.35.0 -xxhash==3.6.0 yarl==1.20.1 diff --git a/src/microbots/MicroBot.py b/src/microbots/MicroBot.py index 7402288f..2c347e9d 100644 --- a/src/microbots/MicroBot.py +++ b/src/microbots/MicroBot.py @@ -1,3 +1,4 @@ +"""Core MicroBot agent class and supporting types.""" from collections.abc import Iterable import json import os @@ -53,6 +54,8 @@ class BotType(StrEnum): + """Enumeration of the supported bot types.""" + READING_BOT = "READING_BOT" WRITING_BOT = "WRITING_BOT" BROWSING_BOT = "BROWSING_BOT" @@ -62,6 +65,19 @@ class BotType(StrEnum): @dataclass class BotRunResult: + """ + Result of a MicroBot run. + + Attributes + ---------- + status : bool + True if the bot completed the task successfully, False otherwise. + result : str | None + The final result/output produced by the bot, or None if unavailable. + error : Optional[str] + An error message if the run failed, or None if it succeeded. + """ + status: bool result: str | None error: Optional[str] @@ -74,7 +90,7 @@ class MicroBot: MicroBot class is the core class representing the autonomous agent. Other bots are extensions of this class. If you want to create a custom bot, you can directly use this class or extend it into your own bot class. - Attributes + Parameters ---------- model : str The model to use for the bot, in the format /. @@ -94,6 +110,10 @@ class MicroBot: can be mounted during the run() method. Refer to `Mount` class regarding the directory structure and permission details. Defaults to None. + token_provider : Optional[any] + A callable that returns a bearer token for Azure AD authentication. + If not provided, it may be auto-created from environment variables. + Defaults to None. """ def __init__( @@ -111,19 +131,19 @@ def __init__( Parameters ---------- - model :str + model : str The model to use for the bot, in the format /. - bot_type :BotType + bot_type : BotType The type of bot being created. It's unused. Will be removed soon. - system_prompt :Optional[str] + system_prompt : Optional[str] The system prompt to guide the bot's behavior. Defaults to None. - environment :Optional[any] + environment : Optional[any] The execution environment for the bot. If not provided, a default LocalDockerEnvironment will be created. - additional_tools :Optional[list[ToolAbstract]] + additional_tools : Optional[list[ToolAbstract]] A list of additional tools to install in the bot's environment. Defaults to None (treated as an empty list). - folder_to_mount :Optional[Mount] + folder_to_mount : Optional[Mount] A folder to mount into the bot's environment. The bot will be given access to this folder based on the specified permissions. This will be the main code folder where the bot will work. Additional folders @@ -132,6 +152,10 @@ def __init__( to None. Note: Supports only mount type MountType.MOUNT for now. + token_provider : Optional[any] + A callable that returns a bearer token for Azure AD authentication. + If not provided, it may be auto-created from environment variables. + Defaults to None. """ self.folder_to_mount = folder_to_mount @@ -203,6 +227,28 @@ def run( max_iterations: int = 20, timeout_in_seconds: int = 200 ) -> BotRunResult: + """ + Run the bot on the given task until completion, timeout, or max iterations. + + Parameters + ---------- + task : str + The task description to give to the bot. + additional_mounts : Optional[list[Mount]] + Additional folders to mount into the bot's environment before + running. Defaults to None. + max_iterations : int + The maximum number of LLM interaction iterations allowed before + aborting the task. Defaults to 20. + timeout_in_seconds : int + The maximum wall-clock time in seconds allowed for the task. + Defaults to 200. + + Returns + ------- + BotRunResult + The outcome of the run, including status, result, and error. + """ if max_iterations <= 0: raise ValueError("max_iterations must be greater than 0") @@ -311,9 +357,18 @@ def run( f" 💭 LLM final thoughts: {llm_response.thoughts}", ) logger.info("🔚 TASK COMPLETED : %s...", task[0:15]) - return BotRunResult(status=True, result=llm_response.thoughts, error=None) + return BotRunResult(status=True, result=llm_response.result or llm_response.thoughts, error=None) def _mount_additional(self, mount: Mount): + """ + Copy an additional folder into the bot's running environment. + + Parameters + ---------- + mount : Mount + The additional mount to copy into the environment. Only + MountType.COPY mounts are supported. + """ if mount.mount_type != MountType.COPY: logger.error( "%s Only COPY mount type is supported for additional mounts for now", @@ -334,6 +389,14 @@ def _mount_additional(self, mount: Mount): # TODO : pass the sandbox path def _create_environment(self, folder_to_mount: Optional[Mount]): + """ + Create the LocalDockerEnvironment for the bot on a free host port. + + Parameters + ---------- + folder_to_mount : Optional[Mount] + The folder to mount into the created environment. + """ free_port = get_free_port() self.environment = LocalDockerEnvironment( @@ -342,6 +405,7 @@ def _create_environment(self, folder_to_mount: Optional[Mount]): ) def _create_llm(self): + """Create the LLM client for the configured model provider.""" # Append tool usage instructions to system prompt system_prompt_with_tools = self.system_prompt if self.system_prompt else "" if self.additional_tools: @@ -370,6 +434,14 @@ def _create_llm(self): # No Else case required as model provider is already validated using _validate_model_and_provider def _validate_model_and_provider(self, model): + """ + Validate that the model string is well-formed and its provider is supported. + + Parameters + ---------- + model : str + The model string in the format /. + """ # Ensure it has only only slash if model.count("/") != 1: raise ValueError("Model should be in the format /") @@ -378,6 +450,14 @@ def _validate_model_and_provider(self, model): raise ValueError(f"Unsupported model provider: {provider}") def _validate_folder_to_mount(self, folder_to_mount: Mount): + """ + Validate that the folder to mount uses a supported mount type. + + Parameters + ---------- + folder_to_mount : Mount + The mount to validate. Only MountType.MOUNT is supported. + """ if folder_to_mount.mount_type != MountType.MOUNT: logger.error( "%s Only MOUNT mount type is supported for folder_to_mount", @@ -388,13 +468,18 @@ def _validate_folder_to_mount(self, folder_to_mount: Mount): ) def _get_dangerous_command_explanation(self, command: str) -> Optional[str]: - """Provides detailed explanation for why a command is dangerous and suggests alternatives. + """ + Provide a detailed explanation for why a command is dangerous and suggest alternatives. - Args: - command: The shell command to analyze + Parameters + ---------- + command : str + The shell command to analyze. - Returns: - str: Explanation with reason and alternative, or None if command is safe + Returns + ------- + Optional[str] + Explanation with reason and alternative, or None if command is safe. """ # Handle invalid commands (empty, None, or non-string) if not command or not isinstance(command, str): @@ -441,19 +526,24 @@ def _get_dangerous_command_explanation(self, command: str) -> Optional[str]: return None def _is_safe_command(self, command: str) -> tuple[bool, Optional[str]]: - """Validates if a command is safe to execute. + """ + Validate whether a command is safe to execute. A command is considered safe if it: - Is not a recursive command (ls -R, rm -rf, tree, find without -maxdepth) - Does not risk generating excessive output or destructive actions - Args: - command: The shell command to validate - - Returns: - tuple[bool, Optional[str]]: A tuple of (is_safe, explanation) where: - - is_safe: True if command is safe to execute, False otherwise - - explanation: Detailed explanation if dangerous, None if safe + Parameters + ---------- + command : str + The shell command to validate. + + Returns + ------- + tuple[bool, Optional[str]] + A tuple of (is_safe, explanation) where is_safe is True if the + command is safe to execute, and explanation is a detailed + explanation if dangerous, or None if safe. """ explanation = self._get_dangerous_command_explanation(command) is_safe = explanation is None diff --git a/src/microbots/auto_memory/__init__.py b/src/microbots/auto_memory/__init__.py new file mode 100644 index 00000000..e967e450 --- /dev/null +++ b/src/microbots/auto_memory/__init__.py @@ -0,0 +1,8 @@ +"""Train <-> eval loop for repo-learning agents. + +Re-exports the public task, outcome, and orchestrator types used to define +an evaluation task and run it in a loop against a training agent. +""" + +from .evalTask import EvalOutcome, EvalTask +from .orchestrator import LoopResult, run_train_eval_loop \ No newline at end of file diff --git a/src/microbots/auto_memory/architecture.md b/src/microbots/auto_memory/architecture.md new file mode 100644 index 00000000..117ce582 --- /dev/null +++ b/src/microbots/auto_memory/architecture.md @@ -0,0 +1,196 @@ +# auto_memory — Architecture + +An agent that **learns a repository into memory notes**, then **proves those notes work** by +solving real SWE-bench issues with them. If it fails, it learns from the failure and retries. + +> Train → Eval → Feedback → Train → … until every instance passes (or rounds run out). + +--- + +## 1. The Big Picture + +```mermaid +flowchart LR + CLI["cli.py
--model --task --config-file --max-rounds"] --> LOOP + + subgraph LOOP["orchestrator: train / eval loop"] + direction TB + E["🎯 EVAL
WritingBot solves each instance
using only the memory notes"] + C{"All resolved?"} + F["🔍 FEEDBACK
ReadingBot combines every
instance result into one message"] + T["🧠 TRAIN
ReadingBot re-reads the repo
and rewrites memory/"] + + E --> C + C -- "yes" --> DONE(["✅ Done"]) + C -- "no" --> F --> T --> E + end +``` + +**Key idea:** the eval agent gets *no* hints beyond the memory notes. +A failing eval is therefore direct evidence the notes are wrong or incomplete. + +--- + +## 2. The Cast + +| File | Role | One-liner | +|---|---|---| +| `cli.py` | Entry point | Resolves the workdir + config, builds the task, calls the orchestrator | +| `orchestrator.py` | Conductor | Clones the training repo, owns the round loop, wires eval ↔ training | +| `evalTask.py` | Contract | `EvalTask`: `parse_config`, `repo_url`, `eval` → `EvalOutcome` | +| `task_registry.py` | Plugin table | `@register_task("name")` + auto-import of everything in `eval/` | +| `eval/swebenchverified.py` | The task | A **set** of SWE-bench-Verified instances, graded by the official harness | +| `training/runner.py` | Trainer | One `ReadingBot` pass with a `MemoryTool` | +| `training/training_instructions.md` | Trainer's brief | "Learn the repo, write notes, never edit code" | +| `workdir.py` | Filing clerk | Every path under `workdir/` lives here — nothing is hard-coded elsewhere | + +--- + +## 3. One Round, Step by Step + +```mermaid +sequenceDiagram + autonumber + participant O as orchestrator + participant W as workdir + participant T as SweBenchVerified + participant One as per-instance task + participant Train as run_training + + O->>W: take_memory_snapshot(round N) + Note over W: copy memory/ ➜ round_N/starting_memory_snapshot + O->>T: eval(memory_dir, model, eval_dir) + + loop every configured instance + T->>One: eval(...) + One->>One: setup() – clone/reset repo @ base commit + One->>One: WritingBot + MemoryTool(memory_dir) + One-->>T: BotRunResult (patch left in the checkout) + T->>One: check() – git diff ➜ SWE-bench harness + One-->>T: resolved / not resolved (+ test_output.txt) + end + + T-->>O: EvalOutcome(passed, score, feedback) + + alt every instance resolved + O-->>O: return LoopResult(passed=True) + else some failed + O->>Train: run_training(feedback, memory_dir) + Train-->>W: memory/ rewritten in place + end + + O->>W: write result.json +``` + +`score` is the **fraction of instances resolved**; `passed` is true only when it reaches `1.0`. +An eval that raises is caught, recorded as `score = -1`, and the loop moves on — one bad +round never discards the rounds before it. + +--- + +## 4. Memory Lifecycle + +Memory is a **directory of markdown notes** with a single home. It is mutated in place; +each round snapshots its starting state so nothing is lost. + +```mermaid +flowchart TD + TOP["workdir/memory/
the notes being optimized"] + S1["round_1/starting_memory_snapshot"] + S2["round_2/starting_memory_snapshot"] + S3["round_N/starting_memory_snapshot"] + + TOP -->|"snapshot at round start"| S1 + S1 -.->|"eval reads, training rewrites"| TOP + TOP -->|"snapshot at round start"| S2 + S2 -.->|"eval reads, training rewrites"| TOP + TOP -->|"snapshot at round start"| S3 + S3 -.->|"eval reads, training rewrites"| TOP +``` + +Rules that matter: + +- **One live copy.** Both the eval agent and the training agent point at `workdir/memory`. +- **Snapshots are read-only history.** `round_N/starting_memory_snapshot` is what round N began with, + so you can diff what a round actually learned. +- **Re-running archives, it doesn't append.** `require_workdir` moves the old workdir to + `workdir_backup_` and carries `task_config.yaml`, `memory/` and `repo/` forward, + so a new run resumes from prior knowledge with clean round output. + +--- + +## 5. Workdir Layout + +```text +workdir/ +├── task_config.yaml # instance_id_list: [...] or repo: django/django +├── repo/ # training checkout, reused across rounds +├── memory/ # the notes being optimized ← the thing under test +└── rounds/round_N/ + ├── starting_memory_snapshot/ # memory/ as it looked when the round began + ├── logs/ # training logs + └── eval/ # owned entirely by the eval task + ├── eval_repo/ # shared checkout, reset per instance + ├── logs/_log.txt + └── result.json +``` + +Two checkouts on purpose: the eval task resets `eval_repo/` for every instance, which would +otherwise destroy the training checkout in `repo/`. + +--- + +## 6. Running It + +```bash +# task_config.yaml selects the eval set, by ID list... +# instance_id_list: +# - django__django-11099 +# ...or by repo: +# repo: django/django + +python -m microbots.auto_memory.cli \ + --model azure-openai/gpt-5.5 \ + --task swebenchverified \ + --workdir ./workdir \ + --max-rounds 5 +``` + +All instances in one config must belong to the **same repo** — that repo is what the training +agent learns, and `SweBenchVerified.repo_url()` derives it from the dataset. + +--- + +## 7. Adding a New Eval Task + +Drop a module in `eval/`. `discover_tasks()` imports everything in that package, so the +`@register_task` decorator fires and the name appears in `--task`. No central factory to edit. + +```python +@register_task("mytask") +class MyTask(EvalTask): + def parse_config(self, config_file: Path) -> None: + ... # load your settings; set self._repo_url or override repo_url() + + def eval(self, memory_dir: str, model: str, eval_dir: str, training_repo_dir: str) -> EvalOutcome: + ... # run every unit of work, return one combined outcome +``` + +Required: `eval`. `parse_config` and `repo_url` have working defaults driven by the config +file's `repo` key. The base `__init__` calls `parse_config` for you — so if you override +`__init__`, initialize your own state *before* calling it. + +--- + +## 8. Failure Handling at a Glance + +| Where it breaks | What happens | +|---|---| +| Agent run raises | Caught per instance; logged; that instance counts as failed | +| Harness never writes `test_output.txt` | `error` falls back to the harness's console output | +| Feedback bot unavailable | Falls back to the raw concatenated per-instance results | +| `task.eval` raises | Logged; recorded as `score = -1`; loop continues to the next round | +| Training raises | Logged; loop continues to the next round without retraining | +| `max_rounds` exhausted | `LoopResult(passed=False)` with every round's outcome | + +Whatever happens, the round's `result.json` is still written. diff --git a/src/microbots/auto_memory/cli.py b/src/microbots/auto_memory/cli.py new file mode 100644 index 00000000..a362a421 --- /dev/null +++ b/src/microbots/auto_memory/cli.py @@ -0,0 +1,101 @@ +"""Command-line entry point for the auto-memory train/eval loop. + +Two modes, selected by ``--task``: + +- ``--task `` given: run the full train <-> eval loop for that + task. + +Both modes are dispatched via ``orchestrator.run``. +""" + +import argparse +import logging +from pathlib import Path + +from microbots.auto_memory.orchestrator import run +from microbots.auto_memory.run_logging import configure_run_logging +from microbots.auto_memory.task_registry import TASK_REGISTRY, discover_tasks +from microbots.auto_memory.workdir import config_path, require_workdir, resolve_workdir + +logger = logging.getLogger(__name__) + +# Import every task module so their @register_task decorators fire. +discover_tasks() + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the CLI's top-level args. + + Task-specific values (e.g. an eval task's instance ID) are not + parsed here; they come from the workdir's config file instead. + + Parameters + ---------- + argv : list[str] | None + Args to parse. Defaults to ``sys.argv[1:]`` when ``None``. + + Returns + ------- + argparse.Namespace + The parsed args. + """ + parser = argparse.ArgumentParser(description="Run the auto-memory train/eval loop.") + parser.add_argument("--model", required=True, help='Model, e.g. "azure-openai/gpt-5.5".') + parser.add_argument( + "--workdir", + help="Directory holding this run's files (repo clone, logs, memory, " + "config). Defaults to './workdir' relative to the current directory.", + ) + parser.add_argument( + "--task", + required=True, + choices=sorted(TASK_REGISTRY), + help="Eval task to run.", + ) + parser.add_argument( + "--config-file", type=Path, help="Path to the task configuration file.", + ) + parser.add_argument("--max-rounds", type=int, default=5) + + return parser.parse_args(argv) + +def main(argv: list[str] | None = None) -> None: + """CLI entry point: run the full train/eval loop. + + Parameters + ---------- + argv : list[str] | None + Args to parse. Defaults to ``sys.argv[1:]`` when ``None``. + """ + args = parse_args(argv) + + # The user can pass either an existing workdir containing a + # task_config.yml file or a task_config.yml file using --config + # option. In the later case, the workdir will be created in + # the default location. + # If both workdir and config options are provided and there exists + # a task_yaml.yml inside the workdir, that file will be ignored and + # the provided --config-file will take precedence. + workdir = Path(args.workdir) if args.workdir else resolve_workdir() + require_workdir(workdir) + configure_run_logging(workdir) + + if not args.config_file: + config_file = config_path(workdir) + else: + config_file = args.config_file + if not config_file.is_file(): + raise FileNotFoundError(f"Config file not found: {config_file}") + + result = run( + workdir=workdir, + model=args.model, + task=TASK_REGISTRY[args.task](config_file=config_file), + max_rounds=args.max_rounds, + ) + if result is not None: + logger.info( + "task=%s passed=%s rounds_run=%d", args.task, result.passed, result.rounds_run + ) + +if __name__ == "__main__": + main() diff --git a/src/microbots/auto_memory/eval/__init__.py b/src/microbots/auto_memory/eval/__init__.py new file mode 100644 index 00000000..f49433aa --- /dev/null +++ b/src/microbots/auto_memory/eval/__init__.py @@ -0,0 +1 @@ +"""Eval tasks discovered and registered for the train <-> eval loop.""" diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py new file mode 100644 index 00000000..7974f46f --- /dev/null +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -0,0 +1,617 @@ +"""SWE-bench-verified eval task. + +Loads instances from the SWE-bench-verified dataset, checks out each +instance's repo at its base commit, has the agent attempt a fix, and +verifies the result via ``swebench.harness.run_evaluation``. +""" + +import json +import shutil +import subprocess +import sys +import tempfile +import uuid +from dataclasses import dataclass +from functools import cache +from logging import getLogger +from pathlib import Path + +import yaml + +from microbots.auto_memory.evalTask import EvalOutcome, EvalTask +from microbots.auto_memory.run_logging import log_to_file +from microbots.auto_memory.task_registry import register_task +from microbots.auto_memory.workdir import eval_log_dir +from microbots.bot.ReadingBot import ReadingBot +from microbots.bot.WritingBot import WritingBot +from microbots.MicroBot import BotRunResult +from microbots.tools.tool_definitions.memory_tool import MemoryTool + +logger = getLogger(__name__) + +SWE_BENCH_VERIFIED = "SWE-bench/SWE-bench_Verified" +EVAL_AGENT_MODEL_NAME = "microbots-eval-agent" + + +@dataclass +class SweBenchInstance: + """A single SWE-bench-verified dataset row. + + Attributes + ---------- + instance_id : str + Unique identifier for the instance, e.g. ``"django__django-11099"``. + repo : str + The GitHub repo this instance belongs to, e.g. ``"django/django"``. + base_commit : str + Commit hash representing the repo state before the issue's fix. + problem_statement : str + The GitHub issue title and body describing the bug to fix. + """ + + instance_id: str + repo: str + base_commit: str + problem_statement: str + + +@cache +def _load_dataset_rows(dataset_name: str): + """Load and cache ``dataset_name``'s ``test`` split for the process's lifetime. + + ``load_dataset`` caches the downloaded files on disk, but still + re-reads and rebuilds the in-memory ``Dataset`` object on every + call. Since ``load_instances_of_repo``/``load_instance_using_id`` + may each be called many times, + this wraps ``load_dataset`` with an in-memory cache keyed by + ``dataset_name``, so the dataset is only loaded once per process. + + Parameters + ---------- + dataset_name : str + Hugging Face dataset name to load. + + Returns + ------- + datasets.Dataset + The loaded ``test`` split. + + Raises + ------ + ImportError + If the optional ``datasets`` package (the ``training`` extra) + isn't installed. + """ + try: + from datasets import load_dataset + except ImportError as exc: + raise ImportError( + "SWE-bench-verified evaluation requires the 'training' extra: " + "pip install 'microbots[training]'" + ) from exc + return load_dataset(dataset_name, split="test") + + +def load_instances_of_repo( + dataset_name: str = SWE_BENCH_VERIFIED, + repo: str | None = None, +) -> list[SweBenchInstance]: + """Load all dataset instances, optionally filtered to a single repo. + + Parameters + ---------- + dataset_name : str + Hugging Face dataset name to load. Defaults to + ``SWE_BENCH_VERIFIED``. + repo : str | None + If given, only instances whose ``repo`` matches this value are + returned, e.g. ``"django/django"``. If ``None``, all instances + are returned. + + Returns + ------- + list[SweBenchInstance] + The matching instances. + """ + rows = _load_dataset_rows(dataset_name) + instances = [ + SweBenchInstance( + instance_id=row["instance_id"], + repo=row["repo"], + base_commit=row["base_commit"], + problem_statement=row["problem_statement"], + ) + for row in rows + if repo is None or row["repo"] == repo + ] + return instances + +def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIFIED) -> SweBenchInstance: + """Load a single dataset instance by its instance ID. + + Parameters + ---------- + instance_id : str + The instance ID to look up, e.g. ``"django__django-11099"``. + dataset_name : str + Hugging Face dataset name to load. Defaults to + ``SWE_BENCH_VERIFIED``. + + Returns + ------- + SweBenchInstance + The matching instance. + + Raises + ------ + ValueError + If no instance with the given ``instance_id`` exists in the + dataset. + """ + rows = _load_dataset_rows(dataset_name) + for row in rows: + if row["instance_id"] == instance_id: + return SweBenchInstance( + instance_id=row["instance_id"], + repo=row["repo"], + base_commit=row["base_commit"], + problem_statement=row["problem_statement"], + ) + raise ValueError(f"instance_id not found: {instance_id}") + +class SweBenchVerifiedTask_one: + """Runs and grades a single SWE-bench-verified instance. + + Checks out the instance's repo at its base commit, gives the agent + the issue's problem statement, and verifies the resulting patch + with the official SWE-bench evaluation harness. + + Parameters + ---------- + instance : SweBenchInstance + The dataset instance this task evaluates against. + """ + + def __init__(self, instance: SweBenchInstance): + """Store the dataset instance this task evaluates against. + + Parameters + ---------- + instance : SweBenchInstance + The dataset instance this task evaluates against. + """ + self.instance = instance + + def setup(self, repo_path: str) -> None: + """Clone the instance's repo, or reset it, to its base commit. + + Parameters + ---------- + repo_path : str + Absolute path to clone (or reset) the repo into. + """ + expected_url = f"https://github.com/{self.instance.repo}.git" + + if Path(repo_path).exists(): + origin = subprocess.run( + ["git", "remote", "get-url", "origin"], + cwd=repo_path, capture_output=True, text=True, + ) + if origin.returncode == 0 and origin.stdout.strip() == expected_url: + subprocess.run( + ["git", "reset", "--hard", self.instance.base_commit], + cwd=repo_path, check=True, + ) + subprocess.run(["git", "clean", "-fd"], cwd=repo_path, check=True) + return + + logger.warning( + "SweBenchVerifiedTask.setup: %s exists but isn't a checkout of %s " + "(origin=%r); removing and re-cloning", + repo_path, expected_url, origin.stdout.strip(), + ) + shutil.rmtree(repo_path) + + subprocess.run(["git", "clone", expected_url, repo_path], check=True) + subprocess.run( + ["git", "checkout", self.instance.base_commit], cwd=repo_path, check=True + ) + + def build_prompt(self) -> str: + """Return the instance's issue text as the agent's prompt. + + Returns + ------- + str + The instance's ``problem_statement``. + """ + return self.instance.problem_statement + + def check(self, repo_path: str, agent_output: str, log_path: str) -> BotRunResult: + """Verify the agent's patch using the SWE-bench evaluation harness. + + Captures the agent's changes as a git diff (after marking any + untracked new files intent-to-add, so files the agent newly + created are included in the diff rather than silently + dropped), submits it as a prediction to + ``swebench.harness.run_evaluation``, and checks whether the + harness marked this instance as resolved. + + Parameters + ---------- + repo_path : str + Absolute path to the repo the agent operated on. + agent_output : str + The agent's raw output/result text. Unused here, since + verification is based on the repo's git diff, not the + agent's textual output. + log_path : str + Path to a log file to append the harness's output to, + including the per-instance ``run_instance.log`` and + ``test_output.txt`` artifacts if the harness produced them + (read before the harness's ``report_dir`` is cleaned up). + + Returns + ------- + BotRunResult + ``status`` is whether the harness marked this instance as + resolved. On failure, ``error`` carries the harness's + ``test_output.txt`` (or its console output, if the harness + died before producing one) so the feedback bot can see why + the tests failed. + """ + subprocess.run( + ["git", "add", "--intent-to-add", "."], cwd=repo_path, check=True + ) + # Diffed against the base commit, so the patch is the same whether or + # not the agent committed its work. + diff = subprocess.run( + ["git", "diff", "--binary", self.instance.base_commit], + cwd=repo_path, + capture_output=True, + text=True, + check=True, + ).stdout + + run_id = f"microbots-{uuid.uuid4().hex[:8]}" + model_name_or_path = EVAL_AGENT_MODEL_NAME + #will need to update when upgraded to ~5.0.2 , removed this flag in the new version + #https://github.com/SWE-bench/SWE-bench/commit/e2c13307b6cf7764a50958b9c8bfbfb3f72cb70a + report_dir = Path(tempfile.mkdtemp()) + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) as pred_file: + pred_path = Path(pred_file.name) + pred_file.write(json.dumps([{ + "instance_id": self.instance.instance_id, + "model_patch": diff, + "model_name_or_path": model_name_or_path, + }])) + + try: + proc = subprocess.run( + [sys.executable, "-m", "swebench.harness.run_evaluation", + "--dataset_name", SWE_BENCH_VERIFIED, + "--max_workers", "1", + "--predictions_path", str(pred_path), + "--run_id", run_id, + "--report_dir", str(report_dir), + "--instance_ids", self.instance.instance_id], + #can add timeout if needed + capture_output=True, text=True, + cwd=report_dir, + ) + #need to update this instance_log_dir path when swebench is upgraded + instance_log_dir = ( + report_dir / "logs" / "run_evaluation" / run_id + / model_name_or_path / self.instance.instance_id + ) + # Read while report_dir still exists; the finally block deletes it. + test_output = "" + with open(log_path, "a") as f: + f.write(proc.stdout + proc.stderr) + for log_filename in ("run_instance.log", "test_output.txt"): + log_file = instance_log_dir / log_filename + if log_file.exists(): + content = log_file.read_text() + if log_filename == "test_output.txt": + test_output = content + f.write(f"\n--- {log_filename} ---\n{content}\n") + + report_file = instance_log_dir / "report.json" + passed = False + if report_file.exists(): + report = json.loads(report_file.read_text()) + passed = report.get(self.instance.instance_id, {}).get("resolved", False) + finally: + pred_path.unlink(missing_ok=True) + shutil.rmtree(report_dir, ignore_errors=True) + + return BotRunResult( + status = passed, + result = "resolved" if passed else "not resolved", + # Harness can fail before producing test_output.txt; fall back to its console output. + error = None if passed else (test_output or proc.stdout + proc.stderr) + ) + + def eval(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> BotRunResult: + """Check out the repo and let the agent attempt the issue. + + Grading is deliberately left to ``check``, which the caller runs + afterwards against the same checkout. + + Parameters + ---------- + repo_path : str + Absolute path to check the instance's repo out into. + memory_dir : str + Directory containing memory files to give the agent via + ``MemoryTool``. + model : str + The model to use, in the format ``/``. + log_path : str + Path to write this instance's log to. Truncated on entry. + + Returns + ------- + BotRunResult + The agent's run result, or a failed result carrying the + exception if the attempt raised. + """ + Path(log_path).parent.mkdir(parents=True, exist_ok=True) + Path(log_path).write_text("") + + try: + self.setup(repo_path) + prompt = self.build_prompt() + bot = WritingBot( + model=model, + folder_to_mount=repo_path, + additional_tools=[MemoryTool(memory_dir=memory_dir, read_only=True)], + ) + bot_result = bot.run( + prompt, + max_iterations=40, + timeout_in_seconds=1800 + ) + + with open(log_path, "a") as f: + f.write(f"Agent output:\n{bot_result.result}\n") + + return bot_result + + except Exception as exc: + logger.exception( + "SweBenchVerifiedTask.eval: iteration raised %s", type(exc).__name__ + ) + with open(log_path, "a") as f: + f.write(f"\nException during eval iteration: {type(exc).__name__}: {exc}\n") + return BotRunResult( + status=False, + result=None, + error=f"{type(exc).__name__}: {exc}" + ) + +@register_task("swebenchverified") +class SweBenchVerified(EvalTask): + """Evaluates memory against a set of SWE-bench-verified instances. + + Every instance in the configured set is attempted with the same + memory, and the round's score is the fraction that the harness + marks resolved. + + Parameters + ---------- + config_file : Path + Path to the task's YAML config file, as consumed by + ``parse_config``. + """ + + def __init__(self, config_file: Path) -> None: + """Load and validate the set of instances to evaluate against. + + Parameters + ---------- + config_file : Path + Path to the task's YAML config file. + """ + # dataset must exist before parse_config populates it. + self.dataset: list[SweBenchInstance] = [] + self.parse_config(config_file=config_file) + + def repo_url(self) -> str: + """Return the clone URL of the repo the instances belong to. + + Returns + ------- + str + The training repo's clone URL. ``parse_config`` guarantees + every instance shares one repo. + """ + return f"https://github.com/{self.dataset[0].repo}.git" + + def parse_config(self, config_file: Path) -> None: + """Load the instances this task evaluates from ``config_file``. + + The YAML file selects instances either by an + ``instance_id_list`` of dataset IDs, or by a ``repo`` naming a + SWE-bench repo such as ``django/django``. + + Parameters + ---------- + config_file : Path + Path to the task's YAML config file. + + Raises + ------ + ValueError + If the selected instances span more than one repo, or if + the config selects no instances at all. + """ + + with open(config_file, "r") as f: + config = yaml.safe_load(f) + + instance_ids = config.get("instance_id_list", []) + repo = config.get("repo", None) + + if instance_ids: + repo = None + for instance_id in instance_ids: + dataset = load_instance_using_id(instance_id) + if not repo: + repo = dataset.repo + elif repo != dataset.repo: + raise ValueError( + f"Conflicting repos for instance_id {instance_id}: {repo} vs {dataset.repo}" + ) + + self.dataset.append(dataset) + + elif repo: + self.dataset = load_instances_of_repo(repo=repo) + + if len(self.dataset) == 0: + raise ValueError("No instances loaded for evaluation.") + + def eval(self, memory_dir: str, model: str, eval_dir: str, training_repo_dir: str) -> EvalOutcome: + """Attempt every configured instance and combine the results. + + Parameters + ---------- + memory_dir : str + Directory containing the memory notes to evaluate. + model : str + The model to use, in the format ``/``. + eval_dir : str + Directory this round's eval owns; holds one checkout and one + log file per instance. + training_repo_dir : str + Absolute path to the persistent training checkout, mounted + for the bot that combines instance results into feedback. + + Returns + ------- + EvalOutcome + ``score`` is the fraction of instances resolved, and + ``passed`` is true only when every one of them was. + """ + eval_path = Path(eval_dir) + # One checkout per instance: the agent's container writes to it as + # root, leaving it unresettable for any instance that came after. + eval_repos_path = eval_path / "eval_repo" + log_dir = eval_log_dir(eval_path) + results = [] + + for instance in self.dataset: + inst_log_path = log_dir / f"{instance.instance_id}_log.txt" + inst_repo_path = eval_repos_path / instance.instance_id + task = SweBenchVerifiedTask_one(instance) + + with log_to_file(inst_log_path): + res = task.eval(str(inst_repo_path), memory_dir, model, str(inst_log_path)) + + if not res.status: + logger.info(f"Evaluation failed for instance {instance.instance_id}: {res.error if res.error else 'Unknown error'}") + results.append(res) + else: + res = task.check(str(inst_repo_path), "", str(inst_log_path)) + results.append(res) + + score = 0 + for result in results: + if result.status: + score += 1 + + score = score / len(self.dataset) + + if score == 1: + feedback = "All evaluations passed." + else: + combine_log_path = log_dir / "combine_result_feedback_log.txt" + with log_to_file(combine_log_path): + feedback = self._combine_result_feedback(results, model, training_repo_dir) + + # NOTE: Let's not teardown the repository as it will be useful for debugging + + return EvalOutcome( + passed = score == 1, + score = score, + feedback = feedback + ) + + def _combine_result_feedback(self, results: list[BotRunResult], model: str, training_repo_dir: str) -> str: + """Summarize every instance's result into one feedback string. + + Parameters + ---------- + results : list[BotRunResult] + One result per attempted instance. + model : str + The model to use, in the format ``/``. + training_repo_dir : str + Absolute path to the persistent training checkout, mounted + for the bot. + + Returns + ------- + str + The bot's summary, falling back to the raw concatenated + results if the bot is unavailable or fails. + """ + + serialized_str = f"Total {len(results)} tests ran and their result and feedback:\n" + + for res in results: + serialized_str += f"\nResult: {'Passed' if res.status else 'Failed'}\n" + serialized_str += f"Optional Feedback: {res.result if res.result else 'None'}\n" + serialized_str += f"Error if there are any: {res.error if res.error else 'None'}\n" + + try: + bot = ReadingBot( + model = model, + folder_to_mount=training_repo_dir + ) + task = f""" + You are combining results from {len(results)} SWE-bench evaluation + runs into ONE feedback report for the next training iteration. The + training agent will read your report to decide what to add or fix + in its memory notes. + + For each result below, note whether it passed or failed. For each + failure, briefly identify the underlying cause (e.g. wrong + file/line targeted, incorrect patch logic, response format error, + timeout) rather than only quoting the raw error. You may open + files under the mounted repo if you need to confirm a root + cause, but do not turn this into a debugging session. + Do not refer to any specific instance or test case by name/ID — + describe causes and guidance in general terms only. + + Then write a report with: + 1. A one-line summary: how many passed vs failed. + 2. Grouped failure patterns: if multiple failures share the same + root cause, describe that cause once rather than repeating + yourself. + 3. Concrete, actionable guidance for the training agent — say + what to change in the memory notes to avoid each failure + pattern next time. Be specific and imperative + (e.g. "Record that config paths must be normalized before + comparison", not "there was a path issue"). + 4. Skip anything about passed cases beyond the summary count; + don't restate their feedback. + + Keep the report tight and skimmable — short paragraphs or bullet + points, no code dumps. Put the final report in the `result` + field once you set task_done=true. + + {serialized_str} + """ + bot_result = bot.run(task=task) + except Exception as e: + logger.warning(f"Combining results failed with exception: {e}") + return f"Combining results failed. raw combined output:\n\n{serialized_str}" + + if bot_result.status: + return bot_result.result if bot_result.result else 'No feedback provided' + else: + return f"Combining results failed. raw combined output:\n\n{serialized_str}" diff --git a/src/microbots/auto_memory/evalTask.py b/src/microbots/auto_memory/evalTask.py new file mode 100644 index 00000000..a10b5490 --- /dev/null +++ b/src/microbots/auto_memory/evalTask.py @@ -0,0 +1,117 @@ +"""Defines the abstract eval task interface for the train <-> eval loop. + +An ``EvalTask`` owns its own config, names the repo the training agent +should learn from, and runs one complete evaluation per round. +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from pathlib import Path + +import yaml + + +@dataclass +class EvalOutcome: + """Result of one round's evaluation. + + Attributes + ---------- + passed : bool + Whether every unit of work in the round passed. + score : float + Fraction of units that passed, or ``-1`` if the round errored. + feedback : str + Text describing what went wrong, fed to the next round's + training pass. + """ + + passed: bool + score: float + feedback: str + +class EvalTask(ABC): + """Base class for an evaluation task in the train <-> eval loop. + + Subclasses must implement ``eval``. ``parse_config`` and + ``repo_url`` have working defaults driven by the config file's + ``repo`` key, and may be overridden by tasks that derive the repo + some other way (see ``SweBenchVerified``). + + Parameters + ---------- + config_file : Path + Path to the task's config file, parsed during initialization. + """ + + _repo_url: str | None = None + + def __init__(self, config_file: Path) -> None: + """Parse the config file and record the training repo URL. + + Parameters + ---------- + config_file : Path + Path to the task's config file. + """ + # NOTE: Don't call this from child class unless you need to reuse + # the parse_config logic from here. + super().__init__() + self.parse_config(config_file=config_file) + + def repo_url(self) -> str: + """Return the URL of the repo the training agent should learn from. + + Returns + ------- + str + The training repo's clone URL. + + Raises + ------ + ValueError + If the config had no ``repo`` key and the subclass did not + override this method. + """ + if not self._repo_url: + raise ValueError("Repo URL is not set in the config file." + " Or you didn't override the base method.") + return self._repo_url + + def parse_config(self, config_file: Path) -> None: + """Read the task's config file, recording the training repo URL. + + Called from ``__init__``. Subclasses that need more than the + ``repo`` key override this to load their own settings too. + + Parameters + ---------- + config_file : Path + Path to the config file to parse. + """ + with open(config_file, "r") as f: + config = yaml.safe_load(f) + self._repo_url = config.get("repo") + + @abstractmethod + def eval(self, memory_dir: str, model: str, eval_dir: str, training_repo_dir: str) -> EvalOutcome: + """Required. Run one full evaluation and return its outcome. + + Parameters + ---------- + memory_dir : str + Directory containing memory files to give the agent via + ``MemoryTool``. + model : str + The model to use, in the format ``/``. + eval_dir : str + Directory this round's eval owns. The task decides what + goes in it (cloned repo, logs, and so on). + training_repo_dir : str + Absolute path to the persistent training checkout. + + Returns + ------- + EvalOutcome + Whether the round passed, its score, and the feedback to use for retraining. + """ diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py new file mode 100644 index 00000000..748e4dd7 --- /dev/null +++ b/src/microbots/auto_memory/orchestrator.py @@ -0,0 +1,272 @@ +"""Orchestrates the train <-> eval loop for repo-learning agents. + +Repeatedly runs an ``EvalTask`` against a repo, and on failure builds +feedback and retrains via the training agent, looping until the task +passes or ``max_rounds`` is exhausted. +""" + +import dataclasses +import json +import shutil +import subprocess +from dataclasses import dataclass, field +from logging import getLogger +from pathlib import Path + +from microbots.auto_memory.evalTask import EvalOutcome, EvalTask +from microbots.auto_memory.run_logging import log_to_file +from microbots.auto_memory.training.runner import run_training +from microbots.auto_memory.workdir import ( + RESULT_FILENAME, + get_eval_dir, + memory_dir, + repo_dir, + take_memory_snapshot, + training_log_path, +) + +logger = getLogger(__name__) + +@dataclass +class LoopResult: + """Result of running ``run_train_eval_loop``. + + Attributes + ---------- + passed : bool + Whether the task passed within ``max_rounds``. + rounds_run : int + Number of eval rounds actually run. + final_outcome : EvalOutcome + The outcome of the last round run. + outcomes : list[EvalOutcome] + The outcome of every round run, in order. + """ + + passed: bool + rounds_run: int + final_outcome: EvalOutcome + outcomes: list[EvalOutcome] = field(default_factory=list) + +def clone_repo(url: str, repo_path: Path) -> None: + """Clone ``url`` into ``repo_path``, or reuse it if already cloned from ``url``. + + Existence alone isn't enough to trust ``repo_path``: it could be an + empty/partial directory left by a previous failed clone, or a + reused workdir whose config now points at a different ``url``. So + if ``repo_path`` exists, its ``origin`` remote is checked against + ``url`` first. Only an exact match is reused as-is; anything else + (mismatched origin, or not a git checkout at all) is removed and + re-cloned, so training never silently runs against missing or + wrong code. + + Parameters + ---------- + url : str + Git URL (or local path) to clone from. + repo_path : Path + Destination directory for the clone. + """ + if repo_path.exists(): + origin = subprocess.run( + ["git", "remote", "get-url", "origin"], + cwd=repo_path, capture_output=True, text=True, + ) + if origin.returncode == 0 and origin.stdout.strip() == url: + return + + logger.warning( + "clone_repo: %s exists but isn't a checkout of %s (origin=%r); " + "removing and re-cloning", + repo_path, url, origin.stdout.strip(), + ) + shutil.rmtree(repo_path) + + subprocess.run(["git", "clone", url, str(repo_path)], check=True) + +def write_eval_result(eval_dir: Path, outcome: EvalOutcome) -> None: + """Write a round's eval result to ``result.json``. + + Parameters + ---------- + eval_dir : Path + The directory for this round's evaluation. + outcome : EvalOutcome + The round's outcome to persist. + """ + path = eval_dir / RESULT_FILENAME + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(dataclasses.asdict(outcome), indent=2)) + + +def run_train_eval_loop( + training_repo_path: str, + workdir: Path, + model: str, + task: EvalTask, + max_rounds: int = 5, +) -> LoopResult: + """Run an eval task in a loop, retraining on failure until it passes. + + Memory lives in one place (``workdir/memory``) and is mutated in + place: each round snapshots it to + ``rounds/round_N/starting_memory_snapshot`` before the eval agent + reads it, so what the round began with stays recoverable. The task + then evaluates against that memory in its own + ``rounds/round_N/eval`` directory. Passing returns immediately; + failing feeds ``outcome.feedback`` to ``run_training``, which + rewrites memory for the next round. Either way the round's outcome + is written to ``result.json``. + + An eval that raises is logged and recorded as a failed outcome so + one bad round cannot discard the rounds before it. + + Parameters + ---------- + training_repo_path : str + Absolute path to the persistent checkout the training agent + reads. Separate from the eval checkout, which the task clones + and resets itself every round. + workdir : Path + This run's workdir (see ``microbots.auto_memory.workdir``). + model : str + The model to use, in the format ``/``. + task : EvalTask + The eval task to run each round. + max_rounds : int + Maximum number of train/eval rounds to attempt. Defaults to 5. + + Returns + ------- + LoopResult + Whether the task passed, how many rounds ran, and every round's + outcome. + + Raises + ------ + ValueError + If ``max_rounds`` is less than 1. + """ + if max_rounds < 1: + raise ValueError(f"max_rounds must be >= 1, got {max_rounds}") + + outcomes: list[EvalOutcome] = [] + + for round_idx in range(1, max_rounds+1): + logger.info( + "run_train_eval_loop: round %d/%d starting", round_idx, max_rounds + ) + mem_dir = memory_dir(Path(workdir)) + eval_dir = get_eval_dir(workdir, round_idx) + take_memory_snapshot(mem_dir, round_idx) + try: + outcome = task.eval(str(mem_dir), model, str(eval_dir), training_repo_path) + outcomes.append(outcome) + except Exception as e: + logger.warning( + "run_train_eval_loop: round %d failed during evaluation;\n" + "Exception: %s\n" + "continuing to next round", + round_idx, e + ) + # Store the failure in outcomes + outcomes.append(EvalOutcome(passed=False, score=-1, feedback=str(e))) + continue + + try: + if outcome.passed: + logger.info( + "run_train_eval_loop: passed on round %d/%d", round_idx, max_rounds + ) + return LoopResult( + passed=True, + rounds_run=round_idx, + final_outcome=outcome, + outcomes=outcomes, + ) + + logger.info( + "run_train_eval_loop: round %d failed (%s), retraining", + round_idx, + outcome.feedback, + ) + + try: + with log_to_file(training_log_path(workdir, round_idx)): + run_training( + repo_path=training_repo_path, + feedback=outcome.feedback, + memory_dir=str(mem_dir), + model=model, + ) + except Exception: + logger.exception( + "run_train_eval_loop: round %d failed to build feedback/retrain; " + "continuing to next round without retraining", + round_idx, + ) + finally: + write_eval_result(eval_dir, outcome) + + logger.info( + "run_train_eval_loop: exhausted %d rounds without passing", max_rounds + ) + return LoopResult( + passed=False, + rounds_run=max_rounds, + final_outcome=outcomes[-1], + outcomes=outcomes, + ) + +def run( + workdir: Path, + model: str, + task: EvalTask, + max_rounds: int = 5, +) -> LoopResult: + """Clone the task's training repo, then run the full train/eval loop. + + Parameters + ---------- + workdir : Path + This run's workdir (see ``microbots.auto_memory.workdir``), + holding the training clone, memory, and all round output. + model : str + The model to use, in the format ``/``. + task : EvalTask + The eval task to run each round. It also supplies the training + repo's clone URL via ``repo_url()``. + max_rounds : int + Maximum number of train/eval rounds to attempt. Defaults to 5. + + Returns + ------- + LoopResult + The eval loop's result. + """ + training_repo_dir = repo_dir(workdir) + clone_repo(task.repo_url(), training_repo_dir) + training_repo_path = str(training_repo_dir) + + # TODO: train-only mode will be implemented if required after proper design + # if task is None: + # # Train-only mode has no rounds of its own; round 1 is just a + # # scratch dir seeded from (and saved back to) top-level memory. + # memory_dir = str(load_round_memory(workdir, 1)) + # run_training_loop( + # repo_path=training_repo_path, + # feedback="", + # memory_dir=memory_dir, + # model=model, + # iterations=training_iterations, + # ) + # save_round_memory(workdir, 1) + # return None + + return run_train_eval_loop( + training_repo_path=training_repo_path, + workdir=workdir, + model=model, + task=task, + max_rounds=max_rounds, + ) \ No newline at end of file diff --git a/src/microbots/auto_memory/run_logging.py b/src/microbots/auto_memory/run_logging.py new file mode 100644 index 00000000..37683b5f --- /dev/null +++ b/src/microbots/auto_memory/run_logging.py @@ -0,0 +1,82 @@ +"""Routes this package's existing log records into a run's workdir files. + +Three sinks, matching the workdir layout: the package's own records go +to ``workdir/log.txt``, while ``log_to_file`` temporarily captures the +agent/tool records emitted underneath a specific step (a training pass, +or one eval instance) into that step's own file. +""" + +import logging +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +from microbots.auto_memory.workdir import log_path + +PACKAGE_LOGGER_NAME = "microbots.auto_memory" +LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s: %(message)s" + + +def _file_handler(path: Path) -> logging.FileHandler: + """Build a formatted file handler appending to ``path``. + + Parameters + ---------- + path : Path + File to append records to. Parent directories are created. + + Returns + ------- + logging.FileHandler + The configured handler. + """ + path.parent.mkdir(parents=True, exist_ok=True) + handler = logging.FileHandler(path, encoding="utf-8") + handler.setFormatter(logging.Formatter(LOG_FORMAT)) + return handler + + +def configure_run_logging(workdir: Path) -> None: + """Send this package's own records to ``workdir/log.txt``. + + Propagation is disabled so these records never also land in the + per-step files that ``log_to_file`` attaches to the root logger. + Call once per run, after ``require_workdir`` has prepared the + workdir, otherwise the log file is archived out from under the + open handle. + + Parameters + ---------- + workdir : Path + The run's workdir. + """ + logger = logging.getLogger(PACKAGE_LOGGER_NAME) + logger.setLevel(logging.INFO) + logger.propagate = False + logger.addHandler(_file_handler(log_path(workdir))) + + +@contextmanager +def log_to_file(path: Path) -> Iterator[None]: + """Capture root-logger records into ``path`` for the duration of the block. + + Used to give a single training pass or eval instance its own file + without changing what any of the underlying agents/tools log. + + Parameters + ---------- + path : Path + File to append records to for the duration of the block. + """ + logger = logging.getLogger() + handler = _file_handler(path) + previous_level = logger.level + if not previous_level or previous_level > logging.INFO: + logger.setLevel(logging.INFO) + logger.addHandler(handler) + try: + yield + finally: + logger.removeHandler(handler) + logger.setLevel(previous_level) + handler.close() diff --git a/src/microbots/auto_memory/task_registry.py b/src/microbots/auto_memory/task_registry.py new file mode 100644 index 00000000..46a6bede --- /dev/null +++ b/src/microbots/auto_memory/task_registry.py @@ -0,0 +1,82 @@ +"""Registry for looking up ``EvalTask`` classes by name. + +Tasks self-register via the ``@register_task`` decorator, so new task +types can be added without editing a central if/elif factory function. +Callers (e.g. a CLI) look a class up in ``TASK_REGISTRY`` and construct +it with the run's config file. +""" + +import importlib +import pkgutil +from collections.abc import Callable + +from microbots.auto_memory.evalTask import EvalTask + +TASK_REGISTRY: dict[str, type[EvalTask]] = {} + +def register_task(name: str) -> Callable[[type[EvalTask]], type[EvalTask]]: + """Register an ``EvalTask`` subclass under ``name`` as a class decorator. + + Each name maps to exactly one class; registering a name twice is a + programming error rather than a silent overwrite. + + Parameters + ---------- + name : str + The key other code will use to look this task up in + ``TASK_REGISTRY``, e.g. ``"swebenchverified"``. + + Returns + ------- + Callable[[type[EvalTask]], type[EvalTask]] + A decorator that registers the class in ``TASK_REGISTRY`` and + returns it unchanged. + """ + + def decorator(task_cls: type[EvalTask]) -> type[EvalTask]: + """Register ``task_cls`` in ``TASK_REGISTRY`` under the enclosing ``name``. + + Parameters + ---------- + task_cls : type[EvalTask] + The ``EvalTask`` subclass to register. + + Returns + ------- + type[EvalTask] + ``task_cls``, unchanged. + + Raises + ------ + ValueError + If ``name`` is already registered to a different class. + """ + registered = TASK_REGISTRY.get(name) + if registered is not None and registered is not task_cls: + raise ValueError( + f"Task name {name!r} is already registered to " + f"{registered.__module__}.{registered.__qualname__}; " + f"cannot also register {task_cls.__module__}.{task_cls.__qualname__}." + ) + TASK_REGISTRY[name] = task_cls + return task_cls + + return decorator + + +def discover_tasks(package_name: str = "microbots.auto_memory.eval") -> None: + """Import every module in ``package_name`` so ``@register_task`` fires. + + Adding a new task only requires dropping a new module into this + package (with its own ``@register_task`` decorator) — no other code + needs to change to make it discoverable. + + Parameters + ---------- + package_name : str + Dotted path of the package to scan for task modules. Defaults + to ``"microbots.auto_memory.eval"``. + """ + package = importlib.import_module(package_name) + for module_info in pkgutil.iter_modules(package.__path__): + importlib.import_module(f"{package_name}.{module_info.name}") diff --git a/src/microbots/auto_memory/training/runner.py b/src/microbots/auto_memory/training/runner.py index 94f6700c..5401f5f2 100644 --- a/src/microbots/auto_memory/training/runner.py +++ b/src/microbots/auto_memory/training/runner.py @@ -3,8 +3,8 @@ from pathlib import Path from microbots.bot.ReadingBot import ReadingBot -from microbots.tools.tool_definitions.memory_tool import MemoryTool from microbots.MicroBot import BotRunResult +from microbots.tools.tool_definitions.memory_tool import MemoryTool _INSTRUCTIONS_PATH = Path(__file__).parent / "training_instructions.md" @@ -14,8 +14,8 @@ def run_training( feedback: str, memory_dir: str, model: str, - max_iterations: int = 20, - timeout_in_seconds: int = 600, + max_iterations: int = 200, + timeout_in_seconds: int = 3600, ) -> BotRunResult: """Run one training pass over a repository and update its memory. @@ -33,9 +33,9 @@ def run_training( Directory in which the memory tool stores its memory. model : str Model identifier used by the reading bot. - max_iterations : int, default=20 + max_iterations : int, default=200 Maximum number of bot iterations. - timeout_in_seconds : int, default=600 + timeout_in_seconds : int, default=3600 Maximum duration of the bot run in seconds. Returns diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py new file mode 100644 index 00000000..047c700a --- /dev/null +++ b/src/microbots/auto_memory/workdir.py @@ -0,0 +1,267 @@ +"""Path/layout helpers for a training run's workdir. + +Centralizes every path this package reads or writes under a run's +``workdir`` (config, repo clone, logs, memory, and per-round/per-eval +outputs), so callers never hard-code layout details themselves. +""" + +import shutil +import time +from pathlib import Path + +WORKDIR_NAME = "workdir" +CONFIG_FILENAME = "task_config.yaml" +REPO_DIRNAME = "repo" +MEMORY_DIRNAME = "memory" +""" +At the beginning of the round, memory from memory_dir will be snapshotted +inside the workdir/rounds/rounds_n/starting_memory_snapshot directory. +So, the final memory after that round should be found at rounds_(n+1) +directory. The memory of the final round will be available at the +main memory dir workdir/memory +""" +STARTING_MEMORY_SNAPSHOT_DIR = "starting_memory_snapshot" +ROUNDS_DIRNAME = "rounds" +EVAL_DIRNAME = "eval" +EVAL_LOG_DIRNAME = "logs" +LOG_FILENAME = "log.txt" +TRAINING_LOG_FILENAME = "training_log.txt" +RESULT_FILENAME = "result.json" + +""" +Expected workdir structure: + + workdir/ + ├── task_config.yaml + ├── log.txt <-- CLI and orchestrator logs + ├── repo/ <-- Training checkout, reused across rounds + ├── memory/ <-- Mutated in place; the run's living memory + └── rounds/round_n/ + ├── training_log.txt <-- Training agent logs for this round + ├── eval/ <-- Managed by the eval task + │ ├── result.json + │ ├── eval_repo/ + │ └── logs/ + │ └── _log.txt <-- One log per eval instance + └── starting_memory_snapshot/ <-- memory/ as it looked when the round began +""" + + +def resolve_workdir(base: Path | None = None) -> Path: + """Resolve the fixed workdir path relative to ``base``. + + Parameters + ---------- + base : Path | None + Directory to resolve ``workdir/`` relative to. Defaults to the + current working directory. + + Returns + ------- + Path + ``workdir`` resolved relative to ``base`` (or ``Path.cwd()``). + """ + workdir = (base or Path.cwd()) / WORKDIR_NAME + return workdir + + +def require_workdir(workdir: Path) -> None: + """Prepare ``workdir`` for a fresh run, archiving any previous one. + + A run must start from clean round output, so an existing workdir is + archived to ``_backup_`` and recreated. The config, + memory, and training checkout are carried over from the archive so + the next run resumes from what the last one learned instead of + re-cloning and re-learning from scratch. + + Parameters + ---------- + workdir : Path + The workdir to prepare. Created if it does not exist. + """ + if workdir.exists(): + backup = workdir.parent / f"{workdir.name}_backup_{int(time.time())}" + shutil.move(str(workdir), str(backup)) + workdir.mkdir(parents=True) + # Moved rather than copied; repo/ can be hundreds of MB. + for name in (CONFIG_FILENAME, MEMORY_DIRNAME, REPO_DIRNAME): + carried_over = backup / name + if carried_over.exists(): + shutil.move(str(carried_over), str(workdir / name)) + + # Every round snapshots this, so it must exist even when empty. + (workdir / MEMORY_DIRNAME).mkdir(parents=True, exist_ok=True) + + +def config_path(workdir: Path) -> Path: + """Return the path to ``workdir``'s config file. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/task_config.yaml``. + """ + return workdir / CONFIG_FILENAME + + +def log_path(workdir: Path) -> Path: + """Return the path to the run's top-level log file. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/log.txt``. + """ + return workdir / LOG_FILENAME + + +def repo_dir(workdir: Path) -> Path: + """Return the path to the training checkout shared across rounds. + + Used only by the retraining step. Eval tasks clone and manage their + own checkout under the round's eval directory, so the two never + collide. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/repo``. + """ + return workdir / REPO_DIRNAME + + +def memory_dir(workdir: Path) -> Path: + """Return the path to the current top-level (latest) memory directory. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/memory``. + """ + return workdir / MEMORY_DIRNAME + + +def take_memory_snapshot(mem_dir: Path, round_idx: int) -> None: + """Snapshot the memory dir as this round's starting point. + + Memory is mutated in place across rounds, so this preserves what + the round started from before training rewrites it. + + Parameters + ---------- + mem_dir : Path + The run's top-level memory directory. + round_idx : int + The 1-based round number. + + Raises + ------ + FileNotFoundError + If ``mem_dir`` does not exist. It may be empty, but the run's + layout must already have created it. + """ + if not mem_dir.exists(): + raise FileNotFoundError(f"Memory directory does not exist: {mem_dir}") + + snapshot_dir = round_dir(mem_dir.parent, round_idx) / STARTING_MEMORY_SNAPSHOT_DIR + if snapshot_dir.exists(): + shutil.rmtree(snapshot_dir) + shutil.copytree(mem_dir, snapshot_dir) + + +def round_dir( + workdir: Path, round_num: int) -> Path: + """Create and return the directory for a training round. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number. + + Returns + ------- + Path + ``workdir/rounds/round_{round_num}`` + """ + path = workdir / ROUNDS_DIRNAME / f"round_{round_num}" + path.mkdir(parents=True, exist_ok=True) + return path + + +def training_log_path(workdir: Path, round_num: int) -> Path: + """Return the path to a round's training log file. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number. + + Returns + ------- + Path + ``workdir/rounds/round_{round_num}/training_log.txt``. + """ + return round_dir(workdir, round_num) / TRAINING_LOG_FILENAME + + +def get_eval_dir( + workdir: Path, round_num: int) -> Path: + """Return the eval task instance's eval directory. Creates it if missing. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number this eval instance belongs to. + + Returns + ------- + Path + ``workdir/rounds/round_{round_num}/eval``. + """ + path = round_dir(workdir, round_num) / EVAL_DIRNAME + path.mkdir(parents=True, exist_ok=True) + return path + + +def eval_log_dir(eval_dir: Path) -> Path: + """Return the directory holding one log file per eval instance. Creates it if missing. + + Parameters + ---------- + eval_dir : Path + The round's eval directory, as returned by ``get_eval_dir``. + + Returns + ------- + Path + ``/logs``. + """ + path = eval_dir / EVAL_LOG_DIRNAME + path.mkdir(parents=True, exist_ok=True) + return path diff --git a/src/microbots/llm/llm.py b/src/microbots/llm/llm.py index 2800790e..a29c9df9 100644 --- a/src/microbots/llm/llm.py +++ b/src/microbots/llm/llm.py @@ -1,3 +1,4 @@ +"""LLM interface and response parsing/validation for MicroBot.""" from dataclasses import dataclass from abc import ABC, abstractmethod import json @@ -16,20 +17,75 @@ @dataclass class LLMAskResponse: + """ + Parsed response from an LLM turn. + + Attributes + ---------- + task_done : bool + Whether the LLM considers the task complete. + thoughts : str + The LLM's reasoning behind its decision. + command : str + The command the LLM wants executed next. + result : str + Optional final result populated by task-specific prompts + (e.g. ReadingBot) when task_done is True. + """ + task_done: bool = False thoughts: str = "" command: str = "" + result: str = "" class LLMInterface(ABC): + """Abstract interface for an LLM client used by MicroBot.""" + @abstractmethod def ask(self, message: str) -> LLMAskResponse: + """ + Send a message to the LLM and return its parsed response. + + Parameters + ---------- + message : str + The message/prompt to send to the LLM. + + Returns + ------- + LLMAskResponse + The parsed LLM response. + """ pass @abstractmethod def clear_history(self) -> bool: + """ + Clear the LLM's conversation history. + + Returns + ------- + bool + True if the history was cleared successfully. + """ pass def _validate_llm_response(self, response: str) -> tuple[bool, LLMAskResponse]: + """ + Validate and parse a raw LLM response string into an LLMAskResponse. + + Parameters + ---------- + response : str + The raw response text returned by the LLM. + + Returns + ------- + tuple[bool, LLMAskResponse] + A tuple of (is_valid, parsed_response). is_valid is False and + parsed_response is None when the response could not be parsed + or failed validation. + """ if self.retries >= self.max_retries: logger.error("Maximum retries reached for LLM response validation.") @@ -43,7 +99,9 @@ def _validate_llm_response(self, response: str) -> tuple[bool, LLMAskResponse]: self.messages.append({"role": "user", "content": "LLM_RES_ERROR: Please respond in the correct JSON format.\n" + llm_output_format_str}) return False, None - if all(key in response_dict for key in LLMAskResponse.__annotations__.keys()): + # "result" is optional, so it's excluded from this required-keys check. + required_keys = ("task_done", "thoughts", "command") + if all(key in response_dict for key in required_keys): logger.info("The llm response is %s ", response_dict) if response_dict.get("task_done") not in [True, False]: @@ -77,6 +135,7 @@ def _validate_llm_response(self, response: str) -> tuple[bool, LLMAskResponse]: task_done=response_dict["task_done"], command=response_dict["command"], thoughts=response_dict.get("thoughts"), + result=response_dict.get("result", ""), ) return True, llm_response else: diff --git a/src/microbots/tools/tool_definitions/memory_tool.py b/src/microbots/tools/tool_definitions/memory_tool.py index 0dd5ec2c..ac737e7c 100644 --- a/src/microbots/tools/tool_definitions/memory_tool.py +++ b/src/microbots/tools/tool_definitions/memory_tool.py @@ -1,3 +1,10 @@ +"""File-backed ``memory`` tool exposed to agents via the text command loop. + +Provides ``MemoryTool``, an ``ExternalTool`` that lets an LLM view, create, +and edit files under a host ``memory_dir`` using ``/memories/...`` paths, +optionally restricted to read-only access. +""" + import argparse import logging import os @@ -18,24 +25,50 @@ class _NoExitArgumentParser(argparse.ArgumentParser): """ArgumentParser that raises ``ValueError`` instead of calling ``sys.exit``.""" def error(self, message: str) -> None: # type: ignore[override] + """Raise ``ValueError`` instead of printing usage and exiting. + + Parameters + ---------- + message : str + The error message argparse would otherwise print. + + Raises + ------ + ValueError + Always raised, carrying ``message``. + """ raise ValueError(message) -INSTRUCTIONS_TO_LLM = """ +# Pieces shared by both the read/write and read-only instructions, so the +# two variants can't drift apart on what they say about `view` and paths. +_VIEW_PROTOCOL_STEP = ( + "ALWAYS run `memory view /memories` BEFORE doing anything else to check " + "for earlier progress." +) + +_VIEW_COMMANDS = """View a file or list a directory: + memory view + memory view --start --end """ + +_VIEW_EXAMPLES = """ memory view /memories + memory view /memories/progress.md --start 1 --end 10""" + +_COMMON_NOTES = """- Paths must start with /memories/. +- In memory view, use --end -1 to read through the end of the file.""" + +INSTRUCTIONS_TO_LLM = f""" Use this tool to persist information to files across steps. All paths must be under /memories/. MEMORY PROTOCOL: -1. ALWAYS run `memory view /memories` BEFORE doing anything else to check for - earlier progress. +1. {_VIEW_PROTOCOL_STEP} 2. Record status, findings and intermediate results as you go. 3. Before completing a task, save your final results to memory. 4. Keep the memory folder organised — rename or delete stale files. ## Commands -View a file or list a directory: - memory view - memory view --start --end +{_VIEW_COMMANDS} Create a file: memory create @@ -57,19 +90,38 @@ def error(self, message: str) -> None: # type: ignore[override] ## Examples - memory view /memories +{_VIEW_EXAMPLES} memory create /memories/progress.md "## Progress\\n- Found bug in src/foo.py line 42" memory str_replace /memories/progress.md --old "line 42" --new "line 45" memory insert /memories/progress.md --line 0 --text "# Task Notes" - memory view /memories/progress.md --start 1 --end 10 memory delete /memories/old_notes.md memory rename /memories/draft.md /memories/final.md ## Notes -- Paths must start with /memories/. +{_COMMON_NOTES} - memory create overwrites if the file already exists. - memory str_replace requires the old text to appear exactly once. -- In memory view, use --end -1 to read through the end of the file. +""" + +READ_ONLY_INSTRUCTIONS_TO_LLM = f""" +Use this tool to read previously recorded memory files. +All paths must be under /memories/. +Only `memory view` is available — this memory store is read-only, so +create/str_replace/insert/delete/rename/clear will all be rejected. + +MEMORY PROTOCOL: +1. {_VIEW_PROTOCOL_STEP} + +## Commands + +{_VIEW_COMMANDS} + +## Examples + +{_VIEW_EXAMPLES} + +## Notes +{_COMMON_NOTES} """ @@ -94,15 +146,46 @@ class MemoryTool(ExternalTool): ) usage_instructions_to_llm: str = Field(default=INSTRUCTIONS_TO_LLM) memory_dir: Optional[str] = Field(default=None) + read_only: bool = Field(default=False) + """ + When true, only ``view`` is permitted; every mutating subcommand + (create/str_replace/insert/delete/rename/clear) is rejected before it + touches the filesystem. Used to give an agent read access to existing + memory without letting it change what future runs see. + """ def __post_init__(self): + """Resolve ``memory_dir``, create it, and finalize read-only defaults. + + Falls back to ``~/.microbots/memory`` when ``memory_dir`` is + unset. When ``read_only`` is true, swaps the default + description/instructions for their read-only equivalents + (leaving any explicitly supplied override untouched), and builds + the command parser. + """ base = Path(self.memory_dir) if self.memory_dir else Path.home() / ".microbots" / "memory" self._memory_dir = base self._memory_dir.mkdir(parents=True, exist_ok=True) + + # Only swap the default copy; an explicitly supplied description/ + # instructions is left alone even in read-only mode. + if self.read_only: + if self.usage_instructions_to_llm == INSTRUCTIONS_TO_LLM: + self.usage_instructions_to_llm = READ_ONLY_INSTRUCTIONS_TO_LLM + if self.description == "File-backed memory store — view, create, edit, delete files under /memories/.": + self.description = "Read-only file-backed memory store — view files under /memories/." self._parser = self._build_parser() def _build_parser(self) -> _NoExitArgumentParser: - """Build the argparse parser with subparsers for each memory subcommand.""" + """Build the argparse parser with subparsers for each memory subcommand. + + Returns + ------- + _NoExitArgumentParser + Parser configured with one subparser per memory subcommand + (``view``, ``create``, ``str_replace``, ``insert``, + ``delete``, ``rename``, ``clear``). + """ parser = _NoExitArgumentParser(prog="memory", add_help=False) subs = parser.add_subparsers(dest="subcommand") @@ -141,7 +224,25 @@ def _build_parser(self) -> _NoExitArgumentParser: # ---------------------------------------------------------------------- # def _resolve(self, path: str) -> Path: - """Resolve a /memories/… path to an absolute host path.""" + """Resolve a /memories/… path to an absolute host path. + + Parameters + ---------- + path : str + An LLM-supplied path, expected to start with ``/memories``. + + Returns + ------- + Path + The absolute host path under ``memory_dir`` that ``path`` + refers to. + + Raises + ------ + ValueError + If ``path`` doesn't start with ``/memories`` or attempts to + traverse outside ``memory_dir``. + """ if not path.startswith("/"): raise ValueError( f"Invalid memory path: {path!r}. Paths must start with /memories/." @@ -175,10 +276,39 @@ def _resolve(self, path: str) -> Path: # ---------------------------------------------------------------------- # def is_invoked(self, command: str) -> bool: + """Return whether ``command`` is a ``memory`` invocation. + + Parameters + ---------- + command : str + The raw command text the agent issued. + + Returns + ------- + bool + True if ``command`` is (or starts with) ``memory``. + """ cmd = command.strip() return cmd == "memory" or cmd.startswith("memory ") def invoke(self, command: str, parent_bot) -> CmdReturn: + """Parse and dispatch a ``memory`` command to its subcommand handler. + + Parameters + ---------- + command : str + The full ``memory ...`` command text. + parent_bot : MicroBot + The bot invoking this tool. Unused; accepted for interface + compatibility with ``ToolAbstract``. + + Returns + ------- + CmdReturn + The subcommand's result, or an error result if parsing + failed, the subcommand is unknown, read-only mode blocks + it, or the handler raised. + """ try: tokens = shlex.split(command) except ValueError as exc: @@ -192,6 +322,13 @@ def invoke(self, command: str, parent_bot) -> CmdReturn: if args.subcommand is None: return CmdReturn(stdout="", stderr="Usage: memory ...", return_code=1) + if self.read_only and args.subcommand != "view": + return CmdReturn( + stdout="", + stderr=f"Memory is read-only: '{args.subcommand}' is not permitted.", + return_code=1, + ) + dispatch = { "view": self._view, "create": self._create, @@ -214,6 +351,19 @@ def invoke(self, command: str, parent_bot) -> CmdReturn: # ---------------------------------------------------------------------- # def _view(self, args: argparse.Namespace) -> CmdReturn: + """Handle ``memory view`` — list a directory or print a file's lines. + + Parameters + ---------- + args : argparse.Namespace + Parsed ``view`` arguments: ``path``, ``start``, ``end``. + + Returns + ------- + CmdReturn + The directory listing or numbered file lines, or an error + result if ``path`` doesn't exist. + """ resolved = self._resolve(args.path) if not resolved.exists(): return CmdReturn(stdout="", stderr=f"Path not found: {args.path!r}", return_code=1) @@ -239,6 +389,19 @@ def _view(self, args: argparse.Namespace) -> CmdReturn: return CmdReturn(stdout=numbered, stderr="", return_code=0) def _create(self, args: argparse.Namespace) -> CmdReturn: + """Handle ``memory create`` — write (or overwrite) a file. + + Parameters + ---------- + args : argparse.Namespace + Parsed ``create`` arguments: ``path``, ``content``. + + Returns + ------- + CmdReturn + Confirmation of the write, or an error result if no + content was supplied. + """ if not args.content: return CmdReturn(stdout="", stderr="Usage: memory create ", return_code=1) content = " ".join(args.content) @@ -249,6 +412,19 @@ def _create(self, args: argparse.Namespace) -> CmdReturn: return CmdReturn(stdout=f"File created: {args.path}", stderr="", return_code=0) def _str_replace(self, args: argparse.Namespace) -> CmdReturn: + """Handle ``memory str_replace`` — replace a unique substring in a file. + + Parameters + ---------- + args : argparse.Namespace + Parsed ``str_replace`` arguments: ``path``, ``old``, ``new``. + + Returns + ------- + CmdReturn + Confirmation of the edit, or an error result if the file + doesn't exist or ``old`` isn't found exactly once. + """ resolved = self._resolve(args.path) if not resolved.is_file(): return CmdReturn(stdout="", stderr=f"File not found: {args.path!r}", return_code=1) @@ -262,6 +438,19 @@ def _str_replace(self, args: argparse.Namespace) -> CmdReturn: return CmdReturn(stdout=f"File {args.path} has been edited.", stderr="", return_code=0) def _insert(self, args: argparse.Namespace) -> CmdReturn: + """Handle ``memory insert`` — insert a line at a given position. + + Parameters + ---------- + args : argparse.Namespace + Parsed ``insert`` arguments: ``path``, ``line``, ``text``. + + Returns + ------- + CmdReturn + Confirmation of the insert, or an error result if the file + doesn't exist or ``line`` is out of range. + """ resolved = self._resolve(args.path) if not resolved.is_file(): return CmdReturn(stdout="", stderr=f"File not found: {args.path!r}", return_code=1) @@ -273,6 +462,19 @@ def _insert(self, args: argparse.Namespace) -> CmdReturn: return CmdReturn(stdout=f"Text inserted at line {args.line} in {args.path}.", stderr="", return_code=0) def _delete(self, args: argparse.Namespace) -> CmdReturn: + """Handle ``memory delete`` — remove a file or directory. + + Parameters + ---------- + args : argparse.Namespace + Parsed ``delete`` arguments: ``path``. + + Returns + ------- + CmdReturn + Confirmation of the deletion, or an error result if + ``path`` is the memory root or doesn't exist. + """ resolved = self._resolve(args.path) if resolved == self._memory_dir.resolve(): return CmdReturn(stdout="", stderr="Cannot delete the /memories root directory", return_code=1) @@ -287,6 +489,20 @@ def _delete(self, args: argparse.Namespace) -> CmdReturn: return CmdReturn(stdout="", stderr=f"Path not found: {args.path!r}", return_code=1) def _rename(self, args: argparse.Namespace) -> CmdReturn: + """Handle ``memory rename`` — move/rename a file or directory. + + Parameters + ---------- + args : argparse.Namespace + Parsed ``rename`` arguments: ``old_path``, ``new_path``. + + Returns + ------- + CmdReturn + Confirmation of the rename, or an error result if either + path is the memory root, the source is missing, or the + destination already exists. + """ old_resolved = self._resolve(args.old_path) new_resolved = self._resolve(args.new_path) memory_root = self._memory_dir.resolve() @@ -304,6 +520,13 @@ def _rename(self, args: argparse.Namespace) -> CmdReturn: return CmdReturn(stdout=f"Renamed {args.old_path} to {args.new_path}.", stderr="", return_code=0) def _clear(self) -> CmdReturn: + """Handle ``memory clear`` — delete and recreate the memory root. + + Returns + ------- + CmdReturn + Confirmation that memory was cleared. + """ if self._memory_dir.exists(): shutil.rmtree(self._memory_dir) self._memory_dir.mkdir(parents=True, exist_ok=True) diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py new file mode 100644 index 00000000..d07f569e --- /dev/null +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -0,0 +1,214 @@ +"""Unit tests for microbots.auto_memory.eval.swebenchverified. + +The SWE-bench dataset, git, the evaluation harness and every bot are +mocked, so these run without network access, Docker or an LLM. +""" + +import json +import subprocess +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import yaml + +from microbots.auto_memory.eval.swebenchverified import ( + SweBenchInstance, + SweBenchVerified, + SweBenchVerifiedTask_one, +) +from microbots.MicroBot import BotRunResult + +MODULE = "microbots.auto_memory.eval.swebenchverified" + +INSTANCE = SweBenchInstance( + instance_id="django__django-11099", + repo="django/django", + base_commit="abc123", + problem_statement="UsernameValidator allows trailing newline", +) + + +def _config(tmp_path: Path, **body) -> Path: + path = tmp_path / "task_config.yaml" + path.write_text(yaml.safe_dump(body)) + return path + + +def _task(tmp_path: Path, **body) -> SweBenchVerified: + with patch(f"{MODULE}.load_instance_using_id", return_value=INSTANCE): + return SweBenchVerified(_config(tmp_path, **body)) + + +# --------------------------------------------------------------------------- +# parse_config / repo_url +# --------------------------------------------------------------------------- + +@pytest.mark.unit +def test_parse_config_loads_the_listed_instances(tmp_path): + task = _task(tmp_path, instance_id_list=["django__django-11099"]) + + assert [i.instance_id for i in task.dataset] == ["django__django-11099"] + assert task.repo_url() == "https://github.com/django/django.git" + + +@pytest.mark.unit +def test_parse_config_loads_every_instance_of_a_repo(tmp_path): + with patch(f"{MODULE}.load_instances_of_repo", return_value=[INSTANCE, INSTANCE]) as loader: + task = SweBenchVerified(_config(tmp_path, repo="django/django")) + + loader.assert_called_once_with(repo="django/django") + assert len(task.dataset) == 2 + + +@pytest.mark.unit +def test_parse_config_rejects_instances_from_different_repos(tmp_path): + other = SweBenchInstance("flask__flask-1", "pallets/flask", "def456", "boom") + + with patch(f"{MODULE}.load_instance_using_id", side_effect=[INSTANCE, other]): + with pytest.raises(ValueError, match="Conflicting repos"): + SweBenchVerified(_config(tmp_path, instance_id_list=["a", "b"])) + + +@pytest.mark.unit +def test_parse_config_rejects_an_empty_selection(tmp_path): + with pytest.raises(ValueError, match="No instances loaded"): + SweBenchVerified(_config(tmp_path, note="nothing selected")) + + +# --------------------------------------------------------------------------- +# check +# --------------------------------------------------------------------------- + +def _fake_harness(report: dict, test_output: str): + """Stand in for the SWE-bench harness, writing its usual artifacts.""" + + def run(cmd, *args, **kwargs): + if cmd[:2] == ["git", "diff"] or cmd[:2] == ["git", "add"]: + return subprocess.CompletedProcess(cmd, 0, stdout="diff --git a b\n", stderr="") + + report_dir = Path(kwargs["cwd"]) + run_id = cmd[cmd.index("--run_id") + 1] + log_dir = ( + report_dir / "logs" / "run_evaluation" / run_id + / "microbots-eval-agent" / INSTANCE.instance_id + ) + log_dir.mkdir(parents=True) + (log_dir / "report.json").write_text(json.dumps(report)) + (log_dir / "test_output.txt").write_text(test_output) + return subprocess.CompletedProcess(cmd, 0, stdout="harness done\n", stderr="") + + return run + + +@pytest.mark.unit +def test_check_passes_when_the_harness_resolves_the_instance(tmp_path): + log_path = tmp_path / "instance.log" + log_path.write_text("") + harness = _fake_harness({INSTANCE.instance_id: {"resolved": True}}, "OK") + + with patch(f"{MODULE}.subprocess.run", side_effect=harness): + result = SweBenchVerifiedTask_one(INSTANCE).check("/repo", "", str(log_path)) + + assert result.status + assert result.result == "resolved" + assert result.error is None + + +@pytest.mark.unit +def test_check_reports_the_test_output_when_the_instance_is_unresolved(tmp_path): + log_path = tmp_path / "instance.log" + log_path.write_text("") + harness = _fake_harness( + {INSTANCE.instance_id: {"resolved": False}}, + "FAILED tests/test_validators.py::test_trailing_newline", + ) + + with patch(f"{MODULE}.subprocess.run", side_effect=harness): + result = SweBenchVerifiedTask_one(INSTANCE).check("/repo", "", str(log_path)) + + assert not result.status + assert "test_trailing_newline" in result.error + assert "test_trailing_newline" in log_path.read_text() + + +# --------------------------------------------------------------------------- +# eval +# --------------------------------------------------------------------------- + +@pytest.mark.unit +def test_eval_scores_the_fraction_of_resolved_instances(tmp_path): + task = _task(tmp_path, instance_id_list=["django__django-11099"]) + task.dataset = [INSTANCE, INSTANCE] + + agent_ran = BotRunResult(status=True, result="patched", error=None) + verdicts = [ + BotRunResult(status=True, result="resolved", error=None), + BotRunResult(status=False, result="not resolved", error="tests failed"), + ] + + with patch.object(SweBenchVerifiedTask_one, "setup"), \ + patch.object(SweBenchVerifiedTask_one, "check", side_effect=verdicts), \ + patch(f"{MODULE}.WritingBot") as writing_bot, \ + patch(f"{MODULE}.MemoryTool"), \ + patch(f"{MODULE}.ReadingBot") as reading_bot: + writing_bot.return_value.run.return_value = agent_ran + reading_bot.return_value.run.return_value = BotRunResult( + status=True, result="one instance still fails", error=None + ) + outcome = task.eval(str(tmp_path / "memory"), "azure-openai/gpt-4o", str(tmp_path / "eval"), str(tmp_path / "repo")) + + assert outcome.score == 0.5 + assert not outcome.passed + assert outcome.feedback == "one instance still fails" + + +@pytest.mark.unit +def test_eval_passes_only_when_every_instance_resolves(tmp_path): + task = _task(tmp_path, instance_id_list=["django__django-11099"]) + resolved = BotRunResult(status=True, result="resolved", error=None) + + with patch.object(SweBenchVerifiedTask_one, "setup"), \ + patch.object(SweBenchVerifiedTask_one, "check", return_value=resolved), \ + patch(f"{MODULE}.WritingBot") as writing_bot, \ + patch(f"{MODULE}.MemoryTool"): + writing_bot.return_value.run.return_value = BotRunResult( + status=True, result="patched", error=None + ) + outcome = task.eval(str(tmp_path / "memory"), "azure-openai/gpt-4o", str(tmp_path / "eval"), str(tmp_path / "repo")) + + assert outcome.passed + assert outcome.score == 1 + assert outcome.feedback == "All evaluations passed." + + +@pytest.mark.unit +def test_eval_skips_the_harness_when_the_agent_itself_failed(tmp_path): + task = _task(tmp_path, instance_id_list=["django__django-11099"]) + + with patch.object(SweBenchVerifiedTask_one, "setup"), \ + patch.object(SweBenchVerifiedTask_one, "check") as check, \ + patch(f"{MODULE}.WritingBot") as writing_bot, \ + patch(f"{MODULE}.MemoryTool"), \ + patch(f"{MODULE}.ReadingBot") as reading_bot: + writing_bot.return_value.run.return_value = BotRunResult( + status=False, result=None, error="agent timed out" + ) + reading_bot.return_value.run.return_value = BotRunResult( + status=True, result="the agent never produced a patch", error=None + ) + outcome = task.eval(str(tmp_path / "memory"), "azure-openai/gpt-4o", str(tmp_path / "eval"), str(tmp_path / "repo")) + + check.assert_not_called() + assert outcome.score == 0 + + +@pytest.mark.unit +def test_combined_feedback_falls_back_to_raw_results_when_the_bot_fails(tmp_path): + task = _task(tmp_path, instance_id_list=["django__django-11099"]) + results = [BotRunResult(status=False, result="not resolved", error="assertion failed")] + + with patch(f"{MODULE}.ReadingBot", side_effect=RuntimeError("no model configured")): + feedback = task._combine_result_feedback(results, "azure-openai/gpt-4o", "/repo") + + assert "assertion failed" in feedback diff --git a/test/auto_memory/test_cli.py b/test/auto_memory/test_cli.py new file mode 100644 index 00000000..be912cff --- /dev/null +++ b/test/auto_memory/test_cli.py @@ -0,0 +1,78 @@ +"""Unit tests for microbots.auto_memory.cli.""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from microbots.auto_memory.cli import main, parse_args + +MODULE = "microbots.auto_memory.cli" + + +@pytest.mark.unit +def test_parse_args_defaults(): + args = parse_args(["--model", "azure-openai/gpt-4o", "--task", "swebenchverified"]) + + assert args.model == "azure-openai/gpt-4o" + assert args.task == "swebenchverified" + assert args.max_rounds == 5 + assert args.workdir is None + + +@pytest.mark.unit +def test_main_builds_the_task_from_the_workdir_config_and_runs_the_loop(tmp_path): + workdir = tmp_path / "workdir" + workdir.mkdir() + (workdir / "task_config.yaml").write_text("repo: django/django") + + task_cls = MagicMock() + + with patch.dict(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": task_cls}, clear=True), \ + patch(f"{MODULE}.run") as mock_run: + main([ + "--model", "azure-openai/gpt-4o", + "--task", "swebenchverified", + "--workdir", str(workdir), + "--max-rounds", "2", + ]) + + task_cls.assert_called_once_with(config_file=workdir / "task_config.yaml") + assert mock_run.call_args.kwargs["max_rounds"] == 2 + assert mock_run.call_args.kwargs["task"] is task_cls.return_value + + +@pytest.mark.unit +def test_main_prefers_an_explicit_config_file(tmp_path): + workdir = tmp_path / "workdir" + workdir.mkdir() + config_file = tmp_path / "elsewhere.yaml" + config_file.write_text("repo: django/django") + + task_cls = MagicMock() + + with patch.dict(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": task_cls}, clear=True), \ + patch(f"{MODULE}.run"): + main([ + "--model", "azure-openai/gpt-4o", + "--task", "swebenchverified", + "--workdir", str(workdir), + "--config-file", str(config_file), + ]) + + task_cls.assert_called_once_with(config_file=config_file) + + +@pytest.mark.unit +def test_main_fails_fast_when_no_config_file_exists(tmp_path): + workdir = tmp_path / "workdir" + workdir.mkdir() + + with patch.dict(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock()}, clear=True), \ + patch(f"{MODULE}.run"): + with pytest.raises(FileNotFoundError): + main([ + "--model", "azure-openai/gpt-4o", + "--task", "swebenchverified", + "--workdir", str(workdir), + ]) diff --git a/test/auto_memory/test_full_loop.py b/test/auto_memory/test_full_loop.py new file mode 100644 index 00000000..7fd108a2 --- /dev/null +++ b/test/auto_memory/test_full_loop.py @@ -0,0 +1,129 @@ +"""End-to-end test of the train <-> eval loop on one SWE-bench instance. + +Only the outside world is mocked: the SWE-bench dataset, git, the +evaluation harness, and the three bots. Everything inside +``microbots.auto_memory`` runs for real, so this exercises the whole +path a live run takes: + + cli.main + -> require_workdir (workdir laid out) + -> SweBenchVerified (config parsed, instance loaded) + -> orchestrator.run (training repo cloned) + -> round 1: eval -> harness says unresolved -> feedback -> training + -> round 2: eval -> harness says resolved -> loop returns + +The instance is unresolved on the first round and resolved on the +second, so both the failure/retrain branch and the success branch are +covered in a single run. +""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import yaml + +from microbots.auto_memory.cli import main +from microbots.auto_memory.eval.swebenchverified import ( + SweBenchInstance, + SweBenchVerifiedTask_one, +) +from microbots.MicroBot import BotRunResult + +SWE_MODULE = "microbots.auto_memory.eval.swebenchverified" + +INSTANCE = SweBenchInstance( + instance_id="django__django-11099", + repo="django/django", + base_commit="abc123", + problem_statement="UsernameValidator allows trailing newline in usernames", +) + + +@pytest.fixture +def workdir(tmp_path): + workdir = tmp_path / "workdir" + workdir.mkdir() + (workdir / "task_config.yaml").write_text( + yaml.safe_dump({"instance_id_list": [INSTANCE.instance_id]}) + ) + return workdir + + +@pytest.mark.integration +def test_full_loop_retrains_after_a_failed_round_then_passes(workdir): + training_bot = MagicMock() + training_bot.run.return_value = BotRunResult(status=True, result="notes written", error=None) + + eval_bot = MagicMock() + eval_bot.run.return_value = BotRunResult(status=True, result="patch applied", error=None) + + feedback_bot = MagicMock() + feedback_bot.run.return_value = BotRunResult( + status=True, result="Memory must cover the username validator regex.", error=None + ) + + # The agent gets it wrong the first round and right the second. + harness_verdicts = [ + BotRunResult(status=False, result="not resolved", error="FAILED test_validators.py"), + BotRunResult(status=True, result="resolved", error=None), + ] + + def fake_clone(url, repo_path): + Path(repo_path).mkdir(parents=True, exist_ok=True) + + def fake_setup(self, repo_path): + Path(repo_path).mkdir(parents=True, exist_ok=True) + + with patch(f"{SWE_MODULE}.load_instance_using_id", return_value=INSTANCE), \ + patch("microbots.auto_memory.orchestrator.clone_repo", side_effect=fake_clone) as clone, \ + patch.object(SweBenchVerifiedTask_one, "setup", fake_setup), \ + patch.object(SweBenchVerifiedTask_one, "check", side_effect=harness_verdicts), \ + patch(f"{SWE_MODULE}.WritingBot", return_value=eval_bot), \ + patch(f"{SWE_MODULE}.ReadingBot", return_value=feedback_bot), \ + patch(f"{SWE_MODULE}.MemoryTool"), \ + patch("microbots.auto_memory.training.runner.ReadingBot", return_value=training_bot), \ + patch("microbots.auto_memory.training.runner.MemoryTool"): + main([ + "--model", "azure-openai/gpt-4o", + "--task", "swebenchverified", + "--workdir", str(workdir), + "--max-rounds", "3", + ]) + + # The training repo is derived from the instance, not from the config. + assert clone.call_args.args[0] == "https://github.com/django/django.git" + + # Two rounds ran, and only the failing one triggered retraining. + assert eval_bot.run.call_count == 2 + training_bot.run.assert_called_once() + assert not (workdir / "rounds" / "round_3").exists() + + # The failed round's feedback is what the training agent was given. + training_prompt = training_bot.run.call_args.args[0] + assert "Memory must cover the username validator regex." in training_prompt + + # Round 1 recorded a failure, round 2 a pass. + round_1 = _result(workdir, 1) + assert round_1["passed"] is False + assert round_1["score"] == 0 + + round_2 = _result(workdir, 2) + assert round_2["passed"] is True + assert round_2["score"] == 1 + + # Each round kept the memory it started from, and the per-instance log. + for round_num in (1, 2): + round_path = workdir / "rounds" / f"round_{round_num}" + assert (round_path / "starting_memory_snapshot").is_dir() + assert ( + round_path / "eval" / "logs" / f"{INSTANCE.instance_id}_log.txt" + ).is_file() + + +def _result(workdir: Path, round_num: int) -> dict: + import json + + return json.loads( + (workdir / "rounds" / f"round_{round_num}" / "eval" / "result.json").read_text() + ) diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py new file mode 100644 index 00000000..b147d58a --- /dev/null +++ b/test/auto_memory/test_orchestrator.py @@ -0,0 +1,116 @@ +"""Unit tests for microbots.auto_memory.orchestrator. + +The training bot is mocked out; these tests only cover how the loop +sequences rounds, reacts to outcomes, and records results. +""" + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from microbots.auto_memory.evalTask import EvalOutcome +from microbots.auto_memory.orchestrator import ( + run_train_eval_loop, + write_eval_result, +) + +MODULE = "microbots.auto_memory.orchestrator" + + +def _outcome(passed: bool, score: float = 0.0, feedback: str = "needs work") -> EvalOutcome: + return EvalOutcome(passed=passed, score=score, feedback=feedback) + + +def _task(*outcomes: EvalOutcome) -> MagicMock: + task = MagicMock() + task.eval.side_effect = list(outcomes) + return task + + +@pytest.fixture +def workdir(tmp_path): + (tmp_path / "memory").mkdir() + return tmp_path + + +@pytest.mark.unit +def test_write_eval_result_serializes_the_outcome(tmp_path): + write_eval_result(tmp_path, _outcome(True, score=1.0, feedback="all good")) + + assert json.loads((tmp_path / "result.json").read_text()) == { + "passed": True, + "score": 1.0, + "feedback": "all good", + } + + +@pytest.mark.unit +def test_loop_stops_on_the_first_passing_round(workdir): + task = _task(_outcome(True, score=1.0)) + + with patch(f"{MODULE}.run_training") as mock_training: + result = run_train_eval_loop("/repo", workdir, "azure-openai/gpt-4o", task, max_rounds=3) + + assert result.passed + assert result.rounds_run == 1 + assert task.eval.call_count == 1 + mock_training.assert_not_called() + + +@pytest.mark.unit +def test_loop_retrains_with_the_round_feedback_then_passes(workdir): + task = _task(_outcome(False, feedback="cover the settings module"), _outcome(True, score=1.0)) + + with patch(f"{MODULE}.run_training") as mock_training: + result = run_train_eval_loop("/repo", workdir, "azure-openai/gpt-4o", task, max_rounds=3) + + assert result.passed + assert result.rounds_run == 2 + mock_training.assert_called_once() + assert mock_training.call_args.kwargs["feedback"] == "cover the settings module" + + +@pytest.mark.unit +def test_loop_gives_up_after_max_rounds(workdir): + task = _task(_outcome(False), _outcome(False)) + + with patch(f"{MODULE}.run_training"): + result = run_train_eval_loop("/repo", workdir, "azure-openai/gpt-4o", task, max_rounds=2) + + assert not result.passed + assert result.rounds_run == 2 + assert len(result.outcomes) == 2 + + +@pytest.mark.unit +def test_a_raising_eval_is_recorded_and_the_loop_continues(workdir): + task = MagicMock() + task.eval.side_effect = [RuntimeError("harness exploded"), _outcome(True, score=1.0)] + + with patch(f"{MODULE}.run_training"): + result = run_train_eval_loop("/repo", workdir, "azure-openai/gpt-4o", task, max_rounds=2) + + assert result.passed + assert result.outcomes[0].score == -1 + assert "harness exploded" in result.outcomes[0].feedback + + +@pytest.mark.unit +def test_each_round_records_its_result_and_memory_snapshot(workdir): + (workdir / "memory" / "notes.md").write_text("what I know") + task = _task(_outcome(False), _outcome(False)) + + with patch(f"{MODULE}.run_training"): + run_train_eval_loop("/repo", workdir, "azure-openai/gpt-4o", task, max_rounds=2) + + for round_num in (1, 2): + round_path = workdir / "rounds" / f"round_{round_num}" + assert (round_path / "eval" / "result.json").is_file() + assert (round_path / "starting_memory_snapshot" / "notes.md").is_file() + + +@pytest.mark.unit +def test_max_rounds_must_be_at_least_one(workdir): + with pytest.raises(ValueError, match="max_rounds"): + run_train_eval_loop("/repo", workdir, "azure-openai/gpt-4o", MagicMock(), max_rounds=0) diff --git a/test/auto_memory/test_task_registry.py b/test/auto_memory/test_task_registry.py new file mode 100644 index 00000000..0389e083 --- /dev/null +++ b/test/auto_memory/test_task_registry.py @@ -0,0 +1,63 @@ +"""Unit tests for microbots.auto_memory.task_registry.""" + +from pathlib import Path + +import pytest + +from microbots.auto_memory.evalTask import EvalOutcome, EvalTask +from microbots.auto_memory.task_registry import ( + TASK_REGISTRY, + discover_tasks, + register_task, +) + + +class _StubTask(EvalTask): + """Minimal concrete task, enough to be registrable.""" + + def parse_config(self, config_file: Path) -> None: + self._repo_url = "https://github.com/acme/widget.git" + + def eval(self, memory_dir: str, model: str, eval_dir: str, training_repo_dir: str) -> EvalOutcome: + return EvalOutcome(passed=True, score=1.0, feedback="ok") + + +@pytest.fixture(autouse=True) +def _restore_registry(): + original = dict(TASK_REGISTRY) + yield + TASK_REGISTRY.clear() + TASK_REGISTRY.update(original) + + +@pytest.mark.unit +def test_register_task_stores_the_class_under_its_name(): + register_task("stub")(_StubTask) + + assert TASK_REGISTRY["stub"] is _StubTask + + +@pytest.mark.unit +def test_registering_the_same_class_twice_is_allowed(): + register_task("stub")(_StubTask) + register_task("stub")(_StubTask) + + assert TASK_REGISTRY["stub"] is _StubTask + + +@pytest.mark.unit +def test_registering_a_second_class_under_one_name_raises(): + class _OtherTask(_StubTask): + pass + + register_task("stub")(_StubTask) + + with pytest.raises(ValueError, match="already registered"): + register_task("stub")(_OtherTask) + + +@pytest.mark.unit +def test_discover_tasks_registers_the_shipped_eval_tasks(): + discover_tasks() + + assert "swebenchverified" in TASK_REGISTRY diff --git a/test/auto_memory/test_workdir.py b/test/auto_memory/test_workdir.py new file mode 100644 index 00000000..29a3a565 --- /dev/null +++ b/test/auto_memory/test_workdir.py @@ -0,0 +1,99 @@ +"""Unit tests for microbots.auto_memory.workdir. + +These are pure filesystem-layout tests; nothing here touches git, an +LLM, or Docker. +""" + +import pytest + +from microbots.auto_memory.workdir import ( + config_path, + get_eval_dir, + memory_dir, + repo_dir, + require_workdir, + resolve_workdir, + round_dir, + take_memory_snapshot, +) + + +@pytest.mark.unit +def test_layout_paths_hang_off_the_workdir(tmp_path): + assert resolve_workdir(tmp_path) == tmp_path / "workdir" + assert config_path(tmp_path) == tmp_path / "task_config.yaml" + assert repo_dir(tmp_path) == tmp_path / "repo" + assert memory_dir(tmp_path) == tmp_path / "memory" + assert round_dir(tmp_path, 2) == tmp_path / "rounds" / "round_2" + assert get_eval_dir(tmp_path, 2) == tmp_path / "rounds" / "round_2" / "eval" + + +@pytest.mark.unit +def test_require_workdir_creates_memory_dir_for_a_new_workdir(tmp_path): + workdir = tmp_path / "workdir" + + require_workdir(workdir) + + assert memory_dir(workdir).is_dir() + + +@pytest.mark.unit +def test_require_workdir_archives_previous_run_and_carries_state_over(tmp_path): + workdir = tmp_path / "workdir" + (workdir / "memory").mkdir(parents=True) + (workdir / "memory" / "notes.md").write_text("learned") + (workdir / "repo").mkdir() + (workdir / "repo" / "code.py").write_text("x = 1") + (workdir / "task_config.yaml").write_text("repo: acme/widget") + (workdir / "rounds" / "round_1").mkdir(parents=True) + + require_workdir(workdir) + + # Config, memory and the training checkout survive. + assert (workdir / "task_config.yaml").read_text() == "repo: acme/widget" + assert (workdir / "memory" / "notes.md").read_text() == "learned" + assert (workdir / "repo" / "code.py").read_text() == "x = 1" + # Previous round output does not. + assert not (workdir / "rounds").exists() + assert any(p.name.startswith("workdir_backup_") for p in tmp_path.iterdir()) + + +@pytest.mark.unit +def test_require_workdir_provides_memory_dir_even_when_previous_run_had_none(tmp_path): + workdir = tmp_path / "workdir" + workdir.mkdir() + (workdir / "task_config.yaml").write_text("repo: acme/widget") + + require_workdir(workdir) + + assert memory_dir(workdir).is_dir() + + +@pytest.mark.unit +def test_round_dir_is_idempotent(tmp_path): + first = round_dir(tmp_path, 1) + (first / "eval").mkdir() + + second = round_dir(tmp_path, 1) + + assert second == first + assert (second / "eval").is_dir(), "re-resolving a round must not wipe it" + + +@pytest.mark.unit +def test_take_memory_snapshot_copies_memory_into_the_round(tmp_path): + mem_dir = memory_dir(tmp_path) + mem_dir.mkdir(parents=True) + (mem_dir / "notes.md").write_text("round one") + + take_memory_snapshot(mem_dir, 1) + (mem_dir / "notes.md").write_text("round two") + + snapshot = round_dir(tmp_path, 1) / "starting_memory_snapshot" / "notes.md" + assert snapshot.read_text() == "round one" + + +@pytest.mark.unit +def test_take_memory_snapshot_requires_the_memory_dir_to_exist(tmp_path): + with pytest.raises(FileNotFoundError): + take_memory_snapshot(memory_dir(tmp_path), 1)