diff --git a/README.md b/README.md index 740d3a0..abf90a9 100644 --- a/README.md +++ b/README.md @@ -336,4 +336,6 @@ python -m http.server 8899 --directory demo/site # 本地预览 默认保持逐步规划;启用 `--execution-mode stage` 后,由慢模型规划阶段、快模型连续选择动作, 规则验证优先,仅在必要时调用模型验证或重规划。 +`--execution-mode goal` 可启用带任务记忆的局部连续执行,阶段结束再独立 Verify; +保留 A11y/CDP 语义融合及 JEV 服务恢复,默认 step/stage 行为不变。 配置、执行边界、开发入口和验收方法见 [阶段执行说明](docs/STAGE_EXECUTION.md)。 diff --git a/docs/RESULT_DIR_CONTRACT.md b/docs/RESULT_DIR_CONTRACT.md index 6867654..65839b2 100644 --- a/docs/RESULT_DIR_CONTRACT.md +++ b/docs/RESULT_DIR_CONTRACT.md @@ -39,6 +39,12 @@ OSWorld 2.1 显式配置替代 judge/用户模拟器时,`code_version.json.res | `result.txt` | 分数(`score\n`) | 同上 | | `episode.json` | `{task_id, harness, stop_reason, steps, status(scored\|failed_unscored), score, exception_type, error?, error_traceback?}` | episode 终态(必写)。`error`/`error_traceback` 仅异常题有:异常消息与栈尾几行(脱敏)——只记类型事后没法定位 | | `timing.json` | `{wall_seconds, status}` | episode 终态 | +| `task_memory.json`(goal) | 最新任务记忆:要求、事实、资料覆盖、交付单元、成果、失败与当前阶段 | goal 事件排水及终态 | +| `evidence/`(goal) | 阶段验证引用的截图/CLI 全文正本;临时目录清理后仍可读取 | 阶段验收/记忆落盘 | + +`timing.json` 另外提供 `environment_seconds`、`model_usage_by_role`、`fast_model`,分别记录环境 +分段时间、逐角色请求时间/token、排序次数/耗时/图像使用次数。缺失 usage 不补零冒充已测值, +另记 `calls_missing_usage`。原字段及 `browser/` 正本规则保持不变。 ## 步号语义(demo 侧对齐依据) @@ -63,7 +69,10 @@ OSWorld 2.1 显式配置替代 judge/用户模拟器时,`code_version.json.res |------|--------|---------| | `task_requirements` | verifier.task_requirements | episode 开始时抽取任务需求(1-12 条) | | `plan_step` | planner.plan_step | 每步规划(截图+可见控件+需求+近期动作→下一步决策) | -| `plan_stage` | stage_planner.plan_stage | 仅 stage 模式:初始阶段、阶段完成或请求修复时生成阶段计划 | +| `plan_stage` | stage_planner.plan_stage / goal_planner.plan_goal | stage/goal 的阶段开始或修复;goal 允许读取已存资料后继续规划 | +| `verify_stage` | goal_verifier.verify_goal | goal 阶段结束时独立验证或提取资料增量;明确执行失败的本地检查不算模型调用 | +| `ground_visual` | grounding.ground_visual | goal 按需生成当前截图的有限候选,不直接执行慢模型坐标 | +| `gui_choice` | planner_choice.PlannerChoicePolicy | 仅显式启用 planner 选择后端时,在相同候选与截图上选 ID;独立记录时间/token,不冒充 JEV 排序分数 | | `verify_observation` | verifier.verify_observation | 每步动作执行后独立验证(BEFORE/AFTER 截图→achieved/not_achieved/uncertain)| `verify_observation` 不信任 planner 的自我宣称:它拿前后截图+预期变化独立裁决,驱动 recent 历史、no_action 连击计数、toggle 检查、以及终态双确认(terminal_claim 模式下逐条核验 requirements + 输出文件存在性)。 @@ -80,6 +89,12 @@ OSWorld 2.1 显式配置替代 judge/用户模拟器时,`code_version.json.res | `stage_local` | 消费已生成的阶段计划;含 stage_goal、milestone 和当帧重绑定后的 plan,不代表新的模型调用 | | `stage_progress` | 里程碑推进,含 stage_goal、milestone、验证 source | | `stage_replan` | 阶段修复请求,含原因及当前里程碑;target_unbound 时另含 `unbound_proposals`、`available_targets`、`recovery_scopes`;不等于调用已经完成 | +| `task_memory` / `memory_update_rejected` | goal 的完整记忆快照/拒绝理由;不是规划模型自行宣称的完成 | +| `goal_action` | 真实回执、当前控件、画面变化与局部效果;effect_verified=false,不等于业务验收通过 | +| `stage_verified` | goal 的阶段裁决、证据引用、结束原因与用时,区分阶段成功和单元完成 | +| `observation_refresh` | 刷新观察而未执行动作,action_replayed=false;不增加动作步数 | +| `recovery` / `goal_stalled` | goal 的视觉候选补全/无可执行进展终止原因 | +| `environment_timing` | setup/observe/execute/export/evaluate 分段耗时 | | `action`(kind 字段) | runner 写的执行记录 `{step, kind: "action", action: {…协议动作…}}` | stage 模式的 planner 事件只在真实规划调用时写入,增加 mode、reason、seconds; diff --git a/infra/config.example.json b/infra/config.example.json index a652093..d1248c1 100644 --- a/infra/config.example.json +++ b/infra/config.example.json @@ -44,6 +44,11 @@ "endpoint_file": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/services/qwen_jev.json", "timeout_seconds": 300 }, + "planner": { + "backend": "planner", + "endpoint": "", + "timeout_seconds": 300 + }, "kev": { "backend": "kev", "endpoint": "", @@ -63,7 +68,14 @@ }, "execution": { "mode": "step", - "stage_max_actions": 24 + "stage_max_actions": 24, + "recovery_max_calls": 2, + "recovery_max_seconds": 300, + "verify_max_tokens": 16384, + "verify_reasoning_effort": null, + "verify_max_chars": 48000, + "image_quality": 85, + "max_request_bytes": 2097152 }, "budget": { "max_steps": 30, diff --git a/pyproject.toml b/pyproject.toml index 1210a05..9024978 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,8 +7,8 @@ name = "tandem" version = "0.1.0" description = "Tandem:快慢双模 CUA harness(主体)+ tandem_eval 集群测评薄适配层" requires-python = ">=3.10" -# 运行时依赖白名单见 AGENTS.md。当前为空:M0 尚无消费者;Pillow 在 M1 截图处理落地时加入。 -dependencies = [] +# 截图处理使用仓库白名单内的 Pillow。 +dependencies = ["Pillow"] [project.optional-dependencies] dev = ["pytest>=7", "ruff>=0.4"] diff --git a/scripts/run_experiment.sh b/scripts/run_experiment.sh index 9705f10..5bce418 100755 --- a/scripts/run_experiment.sh +++ b/scripts/run_experiment.sh @@ -9,9 +9,12 @@ EXPERIMENT="" TASKS="" CONCURRENCY=4 SLOW_MODEL="kimi-k3" +FAST_MODEL="" SLOW_KEY_IDS="1,2,3,4" MAX_STEPS=500 MAX_WALL=14400 +EXECUTION_MODE=stage +STAGE_MAX_ACTIONS=10 DRY_RUN=0 usage() { @@ -19,7 +22,8 @@ usage() { Usage: bash scripts/run_experiment.sh \ --experiment NAME --tasks ID1,ID2,... [--concurrency N] \ - [--slow-model NAME] [--slow-key-ids 1,2,3,4] \ + [--slow-model NAME] [--fast-model NAME] [--slow-key-ids 1,2,3,4] \ + [--execution-mode step|stage|goal] [--stage-max-actions N] \ [--max-steps N] [--max-wall SECONDS] [--dry-run] Experiment names use lowercase letters, digits and hyphens only. @@ -33,9 +37,12 @@ while [[ $# -gt 0 ]]; do --tasks) TASKS="$2"; shift 2 ;; --concurrency) CONCURRENCY="$2"; shift 2 ;; --slow-model) SLOW_MODEL="$2"; shift 2 ;; + --fast-model) FAST_MODEL="$2"; shift 2 ;; --slow-key-ids) SLOW_KEY_IDS="$2"; shift 2 ;; --max-steps) MAX_STEPS="$2"; shift 2 ;; --max-wall) MAX_WALL="$2"; shift 2 ;; + --execution-mode) EXECUTION_MODE="$2"; shift 2 ;; + --stage-max-actions) STAGE_MAX_ACTIONS="$2"; shift 2 ;; --dry-run) DRY_RUN=1; shift ;; -h|--help) usage; exit 0 ;; *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;; @@ -50,6 +57,10 @@ if [[ -z "$TASKS" || "$CONCURRENCY" -lt 1 || "$MAX_STEPS" -lt 1 || "$MAX_WALL" - usage >&2 exit 2 fi +if [[ ! "$EXECUTION_MODE" =~ ^(step|stage|goal)$ || "$STAGE_MAX_ACTIONS" -lt 1 ]]; then + usage >&2 + exit 2 +fi EXPERIMENT_ROOT="${TANDEM_EXPERIMENT_ROOT:-$REPO/rjob_logs}" EXPERIMENT_DIR="$EXPERIMENT_ROOT/$EXPERIMENT" @@ -77,6 +88,7 @@ print(json.loads(Path(sys.argv[1]).read_text())["log"]["results_root"]) PY ) export STAGING EXPERIMENT DRIVER_DIR TASKS CONCURRENCY SLOW_MODEL SLOW_KEY_IDS MAX_STEPS MAX_WALL +export EXECUTION_MODE STAGE_MAX_ACTIONS FAST_MODEL python3 - <<'PY' import json import os @@ -90,9 +102,12 @@ args = { "tasks": os.environ["TASKS"].split(","), "concurrency": int(os.environ["CONCURRENCY"]), "slow_model": os.environ["SLOW_MODEL"], + "fast_model": os.environ["FAST_MODEL"] or None, "slow_key_ids": [int(item) for item in os.environ["SLOW_KEY_IDS"].split(",")], "max_steps": int(os.environ["MAX_STEPS"]), "max_wall_seconds": float(os.environ["MAX_WALL"]), + "execution_mode": os.environ["EXECUTION_MODE"], + "stage_max_actions": int(os.environ["STAGE_MAX_ACTIONS"]), "watch_timeout_seconds": float(os.environ["MAX_WALL"]) + 600, "git_describe": subprocess.check_output( ["git", "describe", "--always", "--dirty"], text=True @@ -123,14 +138,17 @@ run_task() { local run_id="${EXPERIMENT}-${task}" local driver_log="${DRIVER_DIR}/${task}.driver.log" local rc=0 + local fast_args=() + if [[ -n "$FAST_MODEL" ]]; then fast_args=(--fast-model "$FAST_MODEL"); fi - tandem-eval rjob run \ + env PYTHONPATH=src python3 -m tandem_eval rjob run \ --run-id "$run_id" \ --benchmark osworld21 \ --tasks "$task" \ - --execution-mode stage \ - --stage-max-actions 10 \ + --execution-mode "$EXECUTION_MODE" \ + --stage-max-actions "$STAGE_MAX_ACTIONS" \ --slow-model "$SLOW_MODEL" \ + "${fast_args[@]}" \ --slow-key-ids "$SLOW_KEY_IDS" \ --max-steps "$MAX_STEPS" \ --max-wall "$MAX_WALL" \ diff --git a/src/tandem/components.py b/src/tandem/components.py index 3df6554..4e4f785 100644 --- a/src/tandem/components.py +++ b/src/tandem/components.py @@ -1,6 +1,6 @@ """组件契约(依赖注入边界,无插件框架、无隐藏分发)。 -在旧九组件上增加 semantic_enricher,统一 slow planner 与 fast selector 看到的元素语义。 -grounder(visual fallback)默认 None,session 侧判空跳过。compile_gui 产出协议动作。""" +semantic_enricher 统一 planner 与 selector 的元素语义;goal 额外注入任务记忆、阶段验证与 +按需视觉补全。step/stage 装配不变。compile_gui 只产出协议动作,不调用 VM。""" from __future__ import annotations @@ -11,12 +11,17 @@ from .config import SessionConfig from .gui.candidates import ActionCandidateBuilder from .gui.compile import gui_action +from .gui.grounding import ground_visual +from .gui.interaction import GuiInteractionController from .gui.proposals import focused_candidates from .gui.semantics import enrich_targets from .gui.targets import A11yTargetBuilder +from .memory.state import TaskMemory from .models.jev import JevOmniPolicy +from .planning.goal_planner import goal_requirements, plan_goal from .planning.planner import plan_step from .planning.stage_planner import plan_stage +from .verification.goal_verifier import verify_goal from .verification.verifier import task_requirements, verify_observation @@ -32,31 +37,45 @@ class Components: focus_candidates: Callable grounder: Callable | None compile_gui: Callable + memory_factory: Callable | None = None + stage_verifier: Callable | None = None + set_deadline: Callable | None = None + interaction_factory: Callable = GuiInteractionController def build_default(*, client, endpoint: str, config: SessionConfig, image_stage: tuple[str, str] | None = None, endpoint_provider=None) -> Components: - if config.fast_backend == "kev": + if config.fast_backend == "planner": + from .models.planner_choice import PlannerChoicePolicy + + selector = PlannerChoicePolicy(client) + elif config.fast_backend == "kev": from .models.kev import KevPolicy selector = KevPolicy(endpoint, timeout_seconds=config.fast_timeout_seconds) else: selector = JevOmniPolicy(endpoint, timeout_seconds=config.fast_timeout_seconds, - image_stage=image_stage, endpoint_provider=endpoint_provider) + image_stage=image_stage, endpoint_provider=endpoint_provider, + joint_actions=config.execution_mode == "goal") return Components( planner=partial( + plan_goal if config.execution_mode == "goal" else plan_stage if config.execution_mode == "stage" else plan_step, client, allow_cli=config.allow_cli and config.route_policy != "gui", route_policy=config.route_policy, ), verifier=partial(verify_observation, client), - requirements=partial(task_requirements, client), + requirements=partial( + goal_requirements if config.execution_mode == "goal" else task_requirements, client), selector=selector, target_builder=A11yTargetBuilder(), semantic_enricher=enrich_targets, candidate_builder=ActionCandidateBuilder(), focus_candidates=focused_candidates, - grounder=None, + grounder=partial(ground_visual, client) if config.execution_mode == "goal" else None, compile_gui=gui_action, + memory_factory=TaskMemory if config.execution_mode == "goal" else None, + stage_verifier=partial(verify_goal, client) if config.execution_mode == "goal" else None, + set_deadline=getattr(client, "set_deadline", None), ) diff --git a/src/tandem/config.py b/src/tandem/config.py index 5786acc..9a5e36c 100644 --- a/src/tandem/config.py +++ b/src/tandem/config.py @@ -17,23 +17,36 @@ class SessionConfig: cli_timeout_seconds: int = 90 execution_mode: str = "step" stage_max_actions: int = 24 + goal_recovery_max_calls: int = 2 + goal_recovery_max_seconds: float = 300.0 # Checked before another grounding call, not an HTTP interrupt. slow_max_tokens: int = 16384 + slow_verify_max_tokens: int = 16384 + slow_verify_reasoning_effort: str | None = None + stage_verify_max_chars: int = 48000 slow_timeout_seconds: float = 120 slow_direct_api: bool = False slow_reasoning_effort: str | None = None # None=服务端默认;off=enable_thinking:false - fast_backend: str = "jev" # "jev"(qwen-jev-choice /rank)| "kev"(Kev System One) + slow_image_quality: int = 85 + slow_max_request_bytes: int = 2 * 1024 * 1024 + fast_backend: str = "jev" # jev / kev;planner 显式复用慢模型 API 选择候选。 fast_timeout_seconds: float = 300 def __post_init__(self): - if self.execution_mode not in {"step", "stage"}: - raise ValueError("execution_mode must be step or stage") + if self.execution_mode not in {"step", "stage", "goal"}: + raise ValueError("execution_mode must be step, stage or goal") if type(self.stage_max_actions) is not int or self.stage_max_actions < 1: raise ValueError("stage_max_actions must be a positive integer") + if type(self.goal_recovery_max_calls) is not int or self.goal_recovery_max_calls < 1: + raise ValueError("goal_recovery_max_calls must be a positive integer") + if self.goal_recovery_max_seconds <= 0: + raise ValueError("goal_recovery_max_seconds must be positive") if self.route_policy not in {"auto", "gui", "cli"}: raise ValueError("route_policy must be auto, gui or cli") if not 1 <= self.cli_timeout_seconds <= 120: raise ValueError("cli_timeout_seconds must be within 1..120 (协议上限)") - if self.fast_backend not in {"jev", "kev"}: - raise ValueError("fast_backend must be jev or kev") + if self.fast_backend not in {"jev", "kev", "planner"}: + raise ValueError("fast_backend must be jev, kev or planner") if self.slow_reasoning_effort not in {None, "off", "low", "medium", "high", "max"}: raise ValueError("slow_reasoning_effort must be null/off/low/medium/high/max") + if self.slow_verify_reasoning_effort not in {None, "off", "low", "medium", "high", "max"}: + raise ValueError("slow_verify_reasoning_effort must be null/off/low/medium/high/max") diff --git a/src/tandem/gui/candidates.py b/src/tandem/gui/candidates.py index 2b6071a..46cbdae 100644 --- a/src/tandem/gui/candidates.py +++ b/src/tandem/gui/candidates.py @@ -30,8 +30,8 @@ "double_click": "The next useful step is to open one observed file or list item with a double click.", "set_slider": "The next useful step is to set a visible slider to the required value.", "keypress": "The next useful step is one supported keyboard navigation or submission command, not text replacement.", - "scroll": "A required control is outside the current viewport, so scrolling is the next useful step.", - "wait": "The required control is still loading, absent, or disabled, so waiting is the only useful next step.", + "scroll": "Scroll within the foreground view to reveal more controls or content.", + "wait": "Wait briefly, then observe the current UI again.", "done": "Every requirement of the current local GUI goal is visibly satisfied.", "visual_fallback": "The required interaction depends on visual or spatial information not represented by the semantic targets.", "return_to_planner": "The current UI state contradicts the assumptions of the current stage and requires replanning.", @@ -103,7 +103,8 @@ def build( def relevance(target: UITarget) -> tuple[int, int]: words = set(re.findall(r"[a-z0-9]+", target.name.lower())) contextual_words = set( - re.findall(r"[a-z0-9]+", " ".join((target.name, *target.parent_context)).lower()) + re.findall(r"[a-z0-9]+", " ".join( + (target.name, *target.parent_context, target.nearby_text)).lower()) ) return ( len(words & hint_words) * 4 + len(contextual_words & context_words), @@ -149,7 +150,8 @@ def candidates(operation: str, values: list[UITarget], description: str) -> list id=f"{operation}:{target.id}", operation=operation, target_id=target.id, - description=description.format(role=target.role.replace("_", " "), name=target.name), + description=description.format(role=target.role.replace("_", " "), name=target.name) + + (f" Context: {target.nearby_text}" if target.nearby_text else ""), source=target.source, metadata={"semantic_key": target.semantic_key}, ) diff --git a/src/tandem/gui/compile.py b/src/tandem/gui/compile.py index c8be177..42fc37e 100644 --- a/src/tandem/gui/compile.py +++ b/src/tandem/gui/compile.py @@ -17,10 +17,14 @@ def gui_action( ) -> GuiAction | WaitAction | None: operation = winner.operation metadata = winner.metadata - if operation in {"click", "double_click"} and target is not None: + if operation in {"click", "double_click", "right_click"} and ( + target is not None or metadata.get("visual_point") + ): + point = metadata.get("visual_point") if target is None else [ + target.x + target.width // 2, target.y + target.height // 2] return GuiAction( action_type=operation, - parameters={"x": target.x + target.width // 2, "y": target.y + target.height // 2}, + parameters={"x": point[0], "y": point[1]}, target_id=winner.target_id, description=winner.description, ) @@ -36,6 +40,10 @@ def gui_action( description=winner.description, ) if operation == "keypress": + if metadata.get("commit_action") and metadata.get("focus_point"): + return GuiAction(action_type="type", target_id=winner.target_id, description=winner.description, + parameters={"text": "\n", "input_mode": "insert", + "click": metadata["focus_point"]}) sequence = metadata.get("key_sequence") or ( [list(KEYS[winner.id])] if winner.id in KEYS else None ) @@ -56,16 +64,22 @@ def gui_action( description=winner.description, ) if operation == "type_text": - click = None + click = metadata.get("visual_point") + text = metadata["text"] press_f2 = False if target is not None and not metadata.get("focused_input"): click = [target.x + target.width // 2, target.y + target.height // 2] press_f2 = target.role in {"table_cell", "grid_cell"} + if target is not None and target.input_type == "date": + return None # Native date segments are separate observed inputs, not a tab-delimited textbox. + if metadata.get("choice_input"): + text = text.rstrip("\r\n") + "\n" return GuiAction( action_type="type", parameters={ - "text": metadata["text"], - "input_mode": metadata.get("input_mode", "replace"), + "text": text, + "input_mode": ("insert" if metadata.get("choice_input") + else metadata.get("input_mode", "replace")), "click": click, "press_f2": press_f2, }, diff --git a/src/tandem/gui/focus.py b/src/tandem/gui/focus.py index 60d7149..b53daca 100644 --- a/src/tandem/gui/focus.py +++ b/src/tandem/gui/focus.py @@ -83,7 +83,6 @@ def walk(element, active=False, menu_path=()): ["esc"], ["alt", path[0][-1].lower()], *[[part.lower()] for part in path[1:]], - *([["enter"]] if role == "menu_item" else []), ], } ) diff --git a/src/tandem/gui/goal_candidates.py b/src/tandem/gui/goal_candidates.py new file mode 100644 index 0000000..bbce26e --- /dev/null +++ b/src/tandem/gui/goal_candidates.py @@ -0,0 +1,500 @@ +"""Current-observation action space for a whole GUI subgoal; no planner click whitelist.""" + +import json +import re +import xml.etree.ElementTree as ET +from datetime import date +from pathlib import Path +from urllib.parse import unquote, urlsplit + +from .candidates import ActionCandidate +from .focus import KEYS, input_and_menu_context +from .targets import _attribute, _local, _pair, _state + + +def matches_field(hint, label): + """Match an intended field to its observed label, not an arbitrary focused control.""" + def words(value): + return set(re.findall(r"\w+", value.casefold())) - {"the", "a", "field", "required"} + + observed = words(label) + intended = words(hint) + return (bool(observed) and (observed <= intended or len(intended) >= 2 and intended <= observed) + and not label.startswith(("Unnamed ", "Focused "))) + + +def launcher_input(value): + """A launcher search is not a website's identically named Search field.""" + return (value.get("scope") == "desktop_launcher" + or "desktop search" in value.get("target_hint", value["label"]).casefold()) + + +def _input_target(value, targets): + """Bind to one best observed field; spatial hints must not override its own label.""" + scores = [] + for target in targets: + choice = value.get("kind") == "choice" and target.role in { + "button", "push_button", "radio_button", "option", "list_item"} + editable = (target.role in {"textbox", "entry", "searchbox", "spin_button"} + or launcher_input(value) and target.role == "text" and target.focused + or "type_text" in target.operations or target.input_type == "select-one") + if not (editable or choice): + continue + if launcher_input(value) and target.application != "gnome-shell": + continue + if (target.role == "spin_button" and target.input_type != "date" + and re.fullmatch(r"\d{4}-\d{2}-\d{2}", value["text"])): + continue # A complete date is not a Day/Month/Year segment or an ordinary numeric spinner. + named = bool(target.name) and not target.name.startswith(("Unnamed ", "Focused ")) + labels = [target.name] if named else [target.description] + if not named and len(target.nearby_text) <= 80 and "\n" not in target.nearby_text: + labels.append(target.nearby_text) + score = 0 + if choice: + wanted = set(re.findall(r"\w+", value["text"].casefold())) + option_name = re.sub(r"\([^)]*\)", "", target.name.casefold()) + observed = set(re.findall(r"\w+", option_name)) + hint = set(re.findall(r"\w+", value.get("target_hint", value["label"]).casefold())) + if wanted and wanted <= observed <= wanted | hint: + score = 6 # Match a specific option, never a container holding the entire form. + for label in labels: + if not label or not editable: + continue + hint = value.get("target_hint", value["label"]) + if not matches_field(hint, label): + continue # Descriptive context words cannot override the explicit intended field. + ignored = {"brief", "required", "enter", "input", "field", "the", "a"} + if (set(re.findall(r"\w+", hint.casefold())) - ignored + == set(re.findall(r"\w+", label.casefold())) - ignored): + score = max(score, 8) + elif matches_field(value["label"], label): + score = max(score, 4) + elif matches_field(hint, label): + score = max(score, 3) + if score: + scores.append((score, target)) + best = max((score for score, _ in scores), default=0) + matching = [target for score, target in scores if score == best] + return matching[0] if len(matching) == 1 else None + + +def input_satisfied(value, target): + """Chrome may omit a URL scheme in the displayed address, unlike ordinary text fields.""" + text = value["text"] + if value.get("kind") == "choice" and target.role in { + "button", "push_button", "radio_button", "option", "list_item"}: + return target.checked is True or target.selected is True + if target.input_type == "select-one": + return target.value.strip().casefold() == text.strip().casefold() + if ("address" in target.name.casefold() and "://" not in target.value + and re.match(r"^(?:https?|file)://", text)): + text = text.split("://", 1)[1] + return target.value == text or value["input_mode"] == "insert" and text in target.value + + +def input_frontier(space, targets, inputs): + """Expose actionable supplied inputs and missing required values on the current page.""" + fields = list(targets) + pending = [c for c in space.candidates() if c.operation == "type_text"] + inputs = [v for v in inputs if not launcher_input(v) or _input_target(v, fields) is not None] + bindings = [(v, _input_target(v, fields)) for v in inputs] + missing = [t.name for t in fields if (t.role in {"entry", "textbox", "text", "spin_button"} + or t.input_type == "select-one") and not t.value + and re.search(r"\brequired\b|\*", t.name + " " + t.description, re.I) + and not any(bound is t for _, bound in bindings)] + unbound = [v["label"] for v, target in bindings if target is None] + conflicts = [t.name for t in fields + if len({v["text"] for v, bound in bindings if bound is t}) > 1] + for target in fields: + if target.input_type == "date" and any( + bound is target and not re.fullmatch(r"\d{4}-\d{2}-\d{2}", v["text"]) + for v, bound in bindings): + conflicts.append(target.name + " requires ISO YYYY-MM-DD") + navigation = next((t for t in fields if t.focused and "address" in t.name.casefold() + and any(bound is t and input_satisfied(v, t) + and re.match(r"^(?:https?|file)://", v["text"]) + for v, bound in bindings)), None) + if pending: + # Keep recovery/navigation choices, but do not advance past a currently unfilled supplied field. + space.controls.pop("done", None) + space.target_heads["click"] = [c for c in space.target_heads["click"] if not re.fullmatch( + r"(?:next(?: step)?|continue|submit(?: claim| form)?)", next( + (t.name for t in targets if t.id == c.target_id), ""), re.I)] + typed_ids = {c.target_id for c in pending if c.target_id} + if navigation: + typed_ids.add(navigation.id) + space.controls.pop("done", None) + # Type actions already focus their own observed field; a separate click adds no prerequisite. + space.target_heads["click"] = [c for c in space.target_heads["click"] if c.target_id not in typed_ids] + satisfied = [v["label"] for v, target in bindings + if target is not None and input_satisfied(v, target)] + return {"next_intent": ("Press ENTER to navigate to the supplied URL in the focused address bar." + if navigation else pending[0].description if pending else ""), + "missing_required_fields": missing, "unbound_inputs": unbound, + "conflicting_fields": conflicts, "satisfied_inputs": satisfied} + + +def scroll_point(tree, viewport): + """Anchor wheel events inside the observed foreground window, not the desktop center.""" + windows = [] + for element in ET.fromstring(tree or "").iter(): + if _local(element.tag) not in {"frame", "dialog"} or _state(element, "active") is not True: + continue + position = _pair(_attribute(element, "screencoord")) + size = _pair(_attribute(element, "size")) + if position and size and size[0] > 0 and size[1] > 0: + windows.append((*position, *size)) + x, y, width, height = windows[-1] if windows else (0, 0, *viewport) + return (max(0, min(viewport[0] - 1, x + width * 3 // 5)), + max(0, min(viewport[1] - 1, y + height // 2))) + + +def _live_dialog(node, root): + """Unknown visibility on a retained portal node is not proof of a blocking dialog.""" + if (_state(node, "modal") is not True or _state(node, "visible") is False + or _state(node, "showing") is False): + return False + positioned = any(_pair(_attribute(e, "screencoord")) and + (size := _pair(_attribute(e, "size"))) and min(size) > 1 for e in node.iter()) + return bool(positioned and (_state(node, "showing") is True + or _state(node, "active") is True + or node.get("name") == root.get("foreground_title"))) + + +def _keys(candidate): + return candidate.metadata.get("key_sequence") or ( + [list(KEYS[candidate.id])] if candidate.id in KEYS else None) + + +def portal_needs_grounding(tree): + """Retained coordinate-free portal nodes require screenshot confirmation, not background clicks.""" + root = ET.fromstring(tree or "") + return next((node.get("name", "") for app in root.iter() if "desktop-portal" in app.get("name", "") + for node in app.iter() if _local(node.tag) == "dialog" and len(node) > 0 + and _state(node, "modal") is True and _state(node, "visible") is not False + and _state(node, "showing") is not False and not _live_dialog(node, root)), "") + + +def foreground_tree(raw_tree): + """GNOME's visible app menu disambiguates stale ACTIVE flags on background windows.""" + root = ET.fromstring(raw_tree) + apps = list(root) + shell = next((a for a in apps if a.get("name") == "gnome-shell"), None) + if shell is None: + return raw_tree + overview = [e for e in shell.iter() if e.get("name") == "Overview"] + search_visible = any( + _state(e, "editable") is True and _state(e, "showing") is True + and _pair(_attribute(e, "screencoord")) and _pair(_attribute(e, "size")) + for panel in overview for e in panel.iter()) + if search_visible: + # X11 still names the old window while GNOME's full-screen overview owns input. + for app in apps: + if app is not shell: + root.remove(app) + shell.set("active", "true") + root.set("foreground_title", "GNOME Overview") + root.set("foreground_class", '["gnome-shell"]') + return ET.tostring(root, encoding="unicode") + # A portal dialog belongs to a separate process; stale X11 focus must not discard it. + portals = [(app, node) for app in apps if "desktop-portal" in app.get("name", "") + for node in app.iter() if _local(node.tag) == "dialog" + and _live_dialog(node, root)] + if portals: + app, dialog = portals[-1] + app.set("active", "true") + root.set("foreground_title", dialog.get("name", "")) + root.set("foreground_class", json.dumps([app.get("name", "")])) + if root.get("foreground_class"): + def normalize(value): + return re.sub(r"[^a-z0-9]", "", value.casefold()) + + def active(element, value): + key = next((k for k in element.attrib if _local(k) == "active"), "active") + element.set(key, "true" if value else "false") + + classes = [normalize(v) for v in json.loads(root.get("foreground_class")) if v] + title = root.get("foreground_title", "") + for app in apps: + matching = (any(normalize(app.get("name", "")).endswith(c) for c in classes) + or bool(title) and any(e.get("name") == title for e in app.iter() + if _local(e.tag) in {"frame", "dialog"})) + uncertain_portal = "desktop-portal" in app.get("name", "") and portal_needs_grounding( + ET.tostring(app, encoding="unicode")) + if app is not shell and not matching and not uncertain_portal: + root.remove(app) + elif app is not shell: + windows = [e for e in app.iter() if _local(e.tag) in {"frame", "dialog"}] + named = [e for e in windows if title and e.get("name") == title] + if named: + active(app, False) + for window in windows: + active(window, window in named) + return ET.tostring(root, encoding="unicode") + names = [] + for element in shell.iter(): + attrs = {k.rsplit("}", 1)[-1]: v for k, v in element.attrib.items()} + position = re.findall(r"-?\d+", attrs.get("screencoord", "")) + if (element.tag.rsplit("}", 1)[-1] == "menu" and attrs.get("name") + and attrs.get("showing") == "true" and len(position) == 2 + and 0 <= int(position[1]) < 32): + names.append(attrs["name"].casefold()) + matches = [app for app in apps if app is not shell and any( + name in e.get("name", "").casefold() for e in app.iter() + if e.tag.rsplit("}", 1)[-1] in {"frame", "dialog"} for name in names)] + if len(matches) != 1: + return raw_tree + for app in apps: + if app is not shell and app is not matches[0]: + root.remove(app) + return ET.tostring(root, encoding="unicode") + + +def goal_candidates(space, targets, tree, inputs, viewport=(1920, 1080)): + _, menus = input_and_menu_context(tree or "") + fields = list(targets) + bindings = [_input_target(value, fields) for value in inputs] + typed = [] + for index, value in enumerate(inputs): + hint = value.get("target_hint", value["label"]) + target = bindings[index] + if target is None: + continue # Ambiguous/unlabelled fields need an observed click or visual grounding first. + if len({v["text"] for v, bound in zip(inputs, bindings, strict=True) if bound is target}) > 1: + continue + if input_satisfied(value, target): + continue + if value.get("kind") == "choice" and target.role in { + "button", "push_button", "radio_button", "option", "list_item"}: + space.target_heads["click"].insert(0, ActionCandidate( + id=f"GOAL_CHOICE:{index}", operation="click", source="goal_input", target_id=target.id, + description=f"Select the supplied option {value['text']!r} on {target.name!r}.", + metadata={"semantic_key": target.semantic_key})) + continue + if target.input_type == "date": + try: + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value["text"]): + continue + date.fromisoformat(value["text"]) + except ValueError: + continue # Ask the planner for an unambiguous date rather than guessing its order. + if len(target.date_order) != 3 or set(target.date_order) != {"year", "month", "day"}: + continue + parts = [t for t in fields if t.role == "spin_button" + and t.name.casefold().split(" ", 1)[0] in target.date_order + and abs(t.y - target.y) <= 8 and target.x - 8 <= t.x < target.x + target.width] + if len(parts) != 3 or {t.name.casefold().split(" ", 1)[0] for t in parts} != { + "year", "month", "day"}: + continue + expected = dict(zip(("year", "month", "day"), value["text"].split("-"), strict=True)) + target = next((t for part in ("year", "month", "day") for t in parts + if t.name.casefold().split(" ", 1)[0] == part and (not t.value.isdigit() + or int(t.value) != int(expected[part]))), None) + if target is None: + continue + value = {**value, "text": expected[target.name.casefold().split(" ", 1)[0]]} + hint = target.name + typed.append(ActionCandidate( + id=f"GOAL_TEXT:{index}", operation="type_text", source="goal_input", target_id=target.id, + description=(f"Select {value['text']!r} in the observed native choice {target.name!r} " + "using keyboard prefix and Enter." if target.input_type == "select-one" else + f"{value['input_mode']} the observed field {target.name!r} with the " + f"supplied {value['label']}: {value['text']!r}"), + metadata={"text": value["text"], "input_mode": value["input_mode"], + "choice_input": target.input_type == "select-one", + "target_hint": hint, "field_label": target.name, + "semantic_key": target.semantic_key}, + )) + space.target_heads["type_text"] = typed + unnamed_entries = [t for t in targets if t.name.startswith(("Unnamed ", "Focused ")) + and t.role in {"text", "textbox", "entry", "searchbox"}][:2] + present = {c.target_id for c in space.target_heads["click"]} + space.target_heads["click"] = [ActionCandidate( + id="FOCUS:" + t.id, operation="click", target_id=t.id, source="a11y", + description=f"Focus the observed {t.name!r} in {' / '.join(t.parent_context)} before typing.", + metadata={"semantic_key": t.semantic_key}) for t in unnamed_entries if t.id not in present + ] + space.target_heads["click"] + native_files = [t for t in targets if t.role in {"canvas", "label"} + and "nautilus" in (t.application + " ".join(t.parent_context)).lower()][:4] + space.target_heads["double_click"].extend(ActionCandidate( + id="OPEN:" + t.id, operation="double_click", target_id=t.id, source=t.source, + description=f"Open the observed file-manager item {t.name!r}.", + metadata={"semantic_key": t.semantic_key}) for t in native_files) + sliders = [t for t in targets if t.role == "slider"][:2] + space.target_heads["set_slider"] = [ActionCandidate( + id=f"SLIDER:{t.id}:{fraction}", operation="set_slider", target_id=t.id, + description=f"Set {t.name!r} to {fraction:.0%} of its range" + + (f" (value {t.range_min + fraction * (t.range_max - t.range_min):.2f})" + if t.range_min is not None and t.range_max is not None else "") + + f"; current observed value: {t.value or 'unknown'}.", + metadata={"fraction": fraction, "semantic_key": t.semantic_key}) + for t in sliders for fraction in (0.1, 0.25, 0.5, 0.75, 0.9) + if (not t.value or t.range_min is None or t.range_max is None + or abs(float(t.value) - (t.range_min + fraction * (t.range_max - t.range_min))) + > max(1e-6, (t.range_max - t.range_min) * 0.02))] + # A slider's default center click is ambiguous and was repeatedly seeking to the same midpoint. + slider_ids = {t.id for t in sliders} + space.target_heads["click"] = [c for c in space.target_heads["click"] if c.target_id not in slider_ids] + space.target_heads["right_click"] = [ActionCandidate( + id="RIGHT:" + c.target_id, operation="right_click", source=c.source, target_id=c.target_id, + description="Open the context menu for " + c.description, + metadata=dict(c.metadata)) for c in space.target_heads["click"][:8]] + sx, sy = scroll_point(tree, viewport) + for candidate in space.target_heads["scroll"]: + candidate.metadata.update(scroll_x=sx, scroll_y=sy) + for menu in menus[:16]: + space.target_heads["keypress"].append(ActionCandidate( + id="MENU_KEY:" + menu["path"], operation="keypress", source="a11y_menu", + description="Open the observed menu path " + menu["path"], + metadata={"key_sequence": menu["key_sequence"]})) + space.target_heads["keypress"].extend(ActionCandidate( + id="GOAL_KEY:" + "+".join(keys), operation="keypress", source="global", + description=description, metadata={"key_sequence": [keys]}) for keys, description in [ + (["ctrl", "s"], "Save the current document."), + (["ctrl", "o"], "Open the application's file chooser for an existing document."), + (["alt", "tab"], "Switch away from the foreground window to another open application."), + (["shift", "f10"], "Open the focused object's context menu."), + (["ctrl", "l"], "Focus the browser or native file-dialog location field."), + (["ctrl", "f"], "Open the current application's Find field."), + (["shift", "tab"], "Move focus to the previous field."), + (["down"], "Move down in the current selection or menu."), + (["up"], "Move up in the current selection or menu."), + (["right"], "Move right in the current selection, calendar or menu."), + (["left"], "Move left in the current selection, calendar or menu."), + (["ctrl", "t"], "Open a separate browser tab, preserving the current tab."), + (["ctrl", "home"], "Move to the beginning of the current document or page."), + (["ctrl", "end"], "Move to the end of the current document or page."), + (["f6"], "Move keyboard focus to the next pane."), + (["space"], "Select or activate the currently focused control."), + ]) + title = ET.fromstring(tree or "").get("foreground_title", "") + if re.match(r"New Tab(?:\s*[-–]|$)", title, re.I) and any( + "address" in t.name.casefold() and _input_target(v, fields) is t + for t in fields for v in inputs): + space.target_heads["click"] = [c for c in space.target_heads["click"] if not any( + t.id == c.target_id and t.scope == "browser_chrome" and t.name.casefold() == "new tab" + for t in targets)] + space.controls = {k: v for k, v in space.controls.items() + if k in {"wait", "done", "visual_fallback", "return_to_planner"}} + space.controls["done"] = ActionCandidate( + id="STAGE_DONE", operation="done", source="stage_control", + description="The current subtask's completion criteria are satisfied. Request ONE independent " + "stage verification; this does not finish the original task.") + space.controls["return_to_planner"].description = ( + "The current subtask needs unavailable information, new reasoning or an unsupported operation. " + "Return control with existing progress preserved; do not repeat an ineffective action.") + return space + + +def modal_frontier(space, targets, tree): + """A real blocking dialog owns interaction; background movement is not useful recovery.""" + root = ET.fromstring(tree or "") + dialogs = [n for n in root.iter() if _local(n.tag) in {"dialog", "alert"} and _live_dialog(n, root)] + visual = next((c.metadata for c in space.candidates() if c.metadata.get("blocking_overlay")), {}) + if not dialogs and not visual: + return {} + dialog = dialogs[-1] if dialogs else None + name = dialog.get("name", "Dialog") if dialog is not None else visual["overlay_name"] + position = _pair(_attribute(dialog, "screencoord")) if dialog is not None else None + size = _pair(_attribute(dialog, "size")) if dialog is not None else None + bounds = [*position, *size] if position and size else visual.get("overlay_bounds") + by_id = {t.id: t for t in targets} + + def inside(candidate): + target = by_id.get(candidate.target_id) + focus = candidate.metadata.get("focus_point") + if focus and bounds: + x, y, width, height = bounds + if not (x <= focus[0] < x + width and y <= focus[1] < y + height): + return False + if candidate.operation == "keypress": + return _keys(candidate) in [ + [[k]] for k in ("esc", "escape", "enter", "tab", "up", "down", "left", "right", "f6", "space") + ] + [[["ctrl", "l"]], [["shift", "tab"]]] + if bounds: + point = candidate.metadata.get("visual_point") + if target: + point = [target.x + target.width // 2, target.y + target.height // 2] + if candidate.operation == "scroll": + point = [candidate.metadata["scroll_x"], candidate.metadata["scroll_y"]] + x, y, width, height = bounds + return bool(point and x <= point[0] < x + width and y <= point[1] < y + height) + return bool(candidate.metadata.get("blocking_overlay") or target and name in target.parent_context) + + for operation in space.target_heads: + space.target_heads[operation] = [c for c in space.target_heads[operation] if inside(c)] + space.controls = {k: c for k, c in space.controls.items() + if k in {"return_to_planner", "visual_fallback"}} + return {"blocking_overlay": name, "overlay_bounds": bounds, + "overlay_kind": visual.get("overlay_kind", "unknown"), + "next_intent": f"The active dialog {name!r} owns input; work inside its visible controls."} + + +def document_mismatch(document, tree, targets): + """Reject observed contradictions, not a website's human title or a generic app title.""" + title = ET.fromstring(tree or "").get("foreground_title", "") + filename = Path(document).name.casefold() + if Path(document).stem.casefold() in title.casefold(): + return False + addresses = [t.value for t in targets if "address" in t.name.casefold() and "://" in t.value] + if Path(document).suffix.casefold() in {".html", ".htm"} and addresses: + return all(Path(unquote(urlsplit(v).path)).name.casefold() != filename for v in addresses) + return bool(re.search(r"\buntitled\b|\S+\.(?:pptx?|docx?|xlsx?|od[ptsg]|pdf)\b", title, re.I)) + + +def application_mismatch(application, tree, known_only=False): + """Only positive app-family contradictions gate editing; ambiguous titles remain unknown.""" + root = ET.fromstring(tree or "") + actual = (root.get("foreground_title", "") + " " + root.get("foreground_class", "")).casefold() + families = ({"wps", "wpp", "et"}, {"libreoffice", "soffice"}, {"chrome", "chromium"}, + {"shotcut"}, {"totem", "videos"}, {"nautilus", "files"}) + wanted = set(re.findall(r"\w+", application.casefold())) + seen = set(re.findall(r"\w+", actual)) + desired = next((f for f in families if f & wanted), None) + current = next((f for f in families if f & seen), None) + if desired and current: + return desired != current + tokens = [w for w in wanted if len(w) >= 4 and w not in { + "classic", "presentation", "office", "desktop", "google"}] + if any(w in actual for w in tokens): + return False + observed = root.get("foreground_class", "").casefold() + return bool(tokens and (current or observed and not known_only and not any( + name in observed for name in ("gnome-shell", "desktop-portal")))) + + +def document_frontier(space, targets, document=None, application=None): + """Permit opening the right document, never typing task content into a contradicted one.""" + by_id = {t.id: t for t in targets} + filename = Path(document).name.casefold() if document else "" + + def navigation(candidate): + target = by_id.get(candidate.target_id) + if candidate.operation == "type_text": + text = candidate.metadata.get("text", "") + return bool(target and (application and target.application == "gnome-shell" + and text.casefold() == application.casefold() + or filename and ("address" in target.name.casefold() or target.scope == "dialog") + and Path(unquote(urlsplit(text).path)).name.casefold() == filename)) + if candidate.operation == "keypress": + return _keys(candidate) in [ + [["ctrl", "l"]], [["ctrl", "o"]], [["esc"]], [["escape"]], [["alt", "tab"]] + ] or candidate.id == "KEYPRESS_ENTER" and any( + t.focused and ("address" in t.name.casefold() or t.scope == "dialog" + or t.application == "gnome-shell") for t in targets) + if candidate.source == "visual_grounder" and candidate.operation in {"click", "double_click"}: + label = candidate.metadata.get("target_label", "").casefold() + return bool(filename and filename in label or application and matches_field(application, label)) + return bool(target and candidate.operation in {"click", "double_click"} and ( + target.scope in {"browser_chrome", "desktop_launcher"} + or target.application == "gnome-shell" + or filename and filename in target.name.casefold() + or application and application.casefold() in target.name.casefold())) + + for operation in space.target_heads: + space.target_heads[operation] = [c for c in space.target_heads[operation] if navigation(c)] + space.controls = {k: c for k, c in space.controls.items() + if k in {"return_to_planner", "visual_fallback"}} + return {"document_recovery": document, "application_recovery": application, + "next_intent": f"Navigate to {application or ''} {document or ''} first. " + "Do not edit the wrong application/document."} diff --git a/src/tandem/gui/grounding.py b/src/tandem/gui/grounding.py new file mode 100644 index 0000000..1e99714 --- /dev/null +++ b/src/tandem/gui/grounding.py @@ -0,0 +1,86 @@ +"""Existing screenshot-grounded target fallback, reused without changing the JEV service.""" + +import base64 +import json +import re + +from PIL import Image + +from .candidates import ActionCandidate +from .goal_candidates import matches_field + + +def ground_visual(client, instruction, plan, screenshot): + with Image.open(screenshot) as image: + width, height = image.size + proposals = plan["action_proposals"] + value = client.json_request([ + {"role": "system", "content": ( + "Ground the immediate GUI intent in the CURRENT screenshot. Propose up to four distinct " + "visible target alternatives, never a screen grid or a sequence. Do not execute anything. " + "Return JSON {blocking_overlay:null|{name,bounds,kind}," + "candidates:[{operation,target,bounds,text,input_mode,target_hint}]}. " + "If a modal or file chooser blocks the task, identify it and propose ONLY its controls. " + "kind=task_dialog when it is the requested interaction (save, open, choose); " + "kind=obstruction only for an unrelated prompt preventing the local objective; " + "otherwise kind=unknown. Do not close a task dialog as a generic recovery. " + "Do not label an ordinary document/window as a blocking overlay. " + "Operations: click, double_click, right_click, type_text, scroll_up, scroll_down. " + "bounds=[x,y,width,height] in " + "ORIGINAL screenshot pixels. Ground only foreground unobscured targets. For type_text copy exact " + "text/input_mode AND target_hint from ONE supplied input, and ground the visible field named " + "by that hint. target must be its observed label/identity, not the text to enter. Never put " + "unrelated text in the currently focused field; otherwise click to focus first. Use [] if no " + "For date/calendar fields, propose opening the calendar, NOT whole-string type_text: " + "the screenshot alone does not establish the native date input's segment order. " + "required target is visible. Do not invent targets or completion evidence." + )}, + {"role": "user", "content": [ + {"type": "text", "text": json.dumps({"task": instruction, "local_plan": plan, + "screenshot_size": [width, height]}, ensure_ascii=False)}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + + base64.b64encode(screenshot.read_bytes()).decode("ascii")}}, + ]}, + ], lambda v: isinstance(v.get("candidates"), list) + and all(isinstance(i, dict) for i in v["candidates"]), role="ground_visual") + allowed_text = {(p.get("text"), p.get("input_mode"), p.get("target_hint", p.get("label", ""))) + for p in proposals if p.get("operation") == "type_text"} + candidates = [] + for item in value["candidates"][:4]: + operation, box = item.get("operation"), item.get("bounds") + if operation not in {"click", "double_click", "right_click", "type_text", "scroll_up", "scroll_down"}: + continue + if not isinstance(box, list) or len(box) != 4 or any(type(v) is not int for v in box): + continue + x, y, w, h = box + if not (0 <= x < x + w <= width and 0 <= y < y + h <= height): + continue + point = [x + w // 2, y + h // 2] + metadata = {"visual_point": point, "visual_bounds": box, "target_label": item.get("target", "")} + overlay = value.get("blocking_overlay") + if isinstance(overlay, dict) and isinstance(overlay.get("name"), str): + bounds = overlay.get("bounds") + if (isinstance(bounds, list) and len(bounds) == 4 and all(type(v) is int for v in bounds) + and 0 <= bounds[0] < bounds[0] + bounds[2] <= width + and 0 <= bounds[1] < bounds[1] + bounds[3] <= height): + metadata.update(blocking_overlay=True, overlay_name=overlay["name"], overlay_bounds=bounds) + metadata["overlay_kind"] = overlay.get("kind", "unknown") + if operation == "type_text": + if (re.fullmatch(r"\d{4}-\d{2}-\d{2}", str(item.get("text", ""))) + and re.search(r"date|time|calendar|日期|时间", str(item.get("target_hint", "")), re.I)): + continue # Native date typing requires observed CDP input_type/date_order. + if ((item.get("text"), item.get("input_mode"), item.get("target_hint")) not in allowed_text + or not matches_field(item.get("target_hint", ""), item.get("target", ""))): + continue + metadata.update(text=item["text"], input_mode=item["input_mode"], + target_hint=item["target_hint"], field_label=item["target"]) + name = f"VISUAL:{operation}:{x}:{y}:{w}:{h}" + if operation.startswith("scroll_"): + name = f"{operation.upper()}:{name}" + operation = "scroll" + metadata.update(scroll_x=point[0], scroll_y=point[1]) + candidates.append(ActionCandidate( + id=name, operation=operation, source="visual_grounder", metadata=metadata, + description=f"{item['operation']} screenshot-visible {item.get('target', 'control')!r} " + f"at box {box}." + (f" Text: {metadata['text']!r}." if "text" in metadata else ""))) + return candidates diff --git a/src/tandem/gui/interaction.py b/src/tandem/gui/interaction.py new file mode 100644 index 0000000..187cbbd --- /dev/null +++ b/src/tandem/gui/interaction.py @@ -0,0 +1,452 @@ +"""Host-owned field/commit state; no model calls, VM execution or semantic success claims.""" + +import json +import re +import xml.etree.ElementTree as ET +from copy import deepcopy +from pathlib import Path + +from PIL import Image, ImageChops, ImageStat + +from .candidates import ActionCandidate +from .goal_candidates import ( + _input_target, + _keys, + application_mismatch, + document_frontier, + input_frontier, + input_satisfied, + launcher_input, + matches_field, + modal_frontier, + portal_needs_grounding, +) +from .semantics import unrepresented_window + + +def visual_changed(before, after, bounds=None): + frames, sizes = [], [] + for path in (before, after): + with Image.open(path) as image: + sizes.append(image.size) + box = ((bounds[0], bounds[1], bounds[0] + bounds[2], bounds[1] + bounds[3]) if bounds + else (0, min(32, image.height // 4), image.width, image.height)) + frames.append(image.crop(box).convert("L").resize((160, 90))) + threshold = 0.1 if bounds else 1.0 + return sizes[0] != sizes[1] or ImageStat.Stat(ImageChops.difference(*frames)).mean[0] > threshold + + +def visual_identity(candidate): + """A field's input attempt does not become new because its detected box shifts a few pixels.""" + if candidate.source == "visual_grounder" and candidate.operation == "type_text": + return "visual_input|" + " ".join(candidate.metadata.get("target_hint", "").casefold().split()) + return None + + +def _binding(value): + return json.dumps([value.get("target_hint", value["label"]).casefold(), value["text"].rstrip()], + ensure_ascii=False) + + +def _command_input(value): + return value.get("kind") == "command" or value.get("commit") == "enter" or bool( + re.search(r"\b(input|command|console)\b", value.get("target_hint", value["label"]), re.I) + and re.match(r"\w+\s*\(", value["text"])) + + +def target_effect(old, targets): + """Observed field/slider effects, never semantic task acceptance.""" + if old["operation"] == "type_text": + focus = [t for t in targets if t.focused and t.role != "password_text" + and (not old.get("field_label") or matches_field(old["field_label"], t.name))] + intended = [t for t in targets if old.get("target_semantic") + and old["target_semantic"] == t.semantic_key] + if not intended and old.get("field_label"): + intended = [t for t in targets if matches_field(old["field_label"], t.name) + and t.role in {"entry", "text", "textbox", "spin_button", "searchbox"}] + if intended: + focus = intended + if len(focus) == 1 and (focus[0].value or old.get("field_label")): + if old.get("field_label") and not matches_field(old["field_label"], focus[0].name): + return "not_met" + matches = input_satisfied( + {"text": old["expected_text"], "input_mode": old["input_mode"]}, focus[0]) + return "met" if matches else "not_met" + if old["operation"] == "set_slider": + target = next((t for t in targets if t.semantic_key == old["target_semantic"]), None) + if (target and target.value and target.range_min is not None and target.range_max is not None + and target.range_max > target.range_min): + fraction = (float(target.value) - target.range_min) / (target.range_max - target.range_min) + return "met" if abs(fraction - old["fraction"]) <= 0.04 else "not_met" + return "unknown" + + +def _content_state(targets): + """Task controls/values, not focus, coordinates, shell clocks or pixel animation.""" + return tuple(sorted((t.role, re.sub(r" - Memory usage - .+$", "", t.name), + "" if t.role == "slider" else t.value, str(t.checked)) for t in targets + if t.application != "gnome-shell" and "gnome-shell" not in t.parent_context + and not t.name.startswith(("Focused ", "Unnamed ")))) + + +def _navigation_family(operation, sequence): + if operation == "scroll": + return "scroll" + if sequence in ([["tab"]], [["shift", "tab"]]): + return "focus_traversal" + if sequence in ([["up"]], [["down"]], [["left"]], [["right"]]): + return "direction_keys" + if sequence in ([["enter"]], [["esc"]], [["escape"]]): + return "activation_keys" + return None + + +class GuiInteractionController: + def __init__(self): + self.stage_id = None + self.fields = {} + self.invalidate_visual = False + self.control = None + self.checkpoint_due = False + self.local_effect = "not_assessed" + self.content_state = () + self.scenes = {} + self.memory = None + self.observation_ref = "" + self.rejected_states = {} + + def receive(self, pending, effect, changed, returncode, *, targets=(), evidence_ref=""): + """An acknowledged dispatch is not proof of the field value or command's success.""" + self.local_effect = "not_assessed" + key = pending.get("interaction_binding") + if key in self.fields: + field = self.fields[key] + if returncode: + field["phase"] = "needs_input" + elif pending.get("commit_action"): + field["phase"] = "submitted_unconfirmed" + elif effect == "not_met": + field["phase"] = "needs_input" + elif field["commit"]: + field["phase"] = "needs_commit" + else: + field["phase"] = "confirmed" if effect == "met" else "typed_unconfirmed" + field.update(observed_change=changed, effect=effect) + sequence = pending.get("key_sequence") + self.invalidate_visual = (pending["operation"] == "scroll" or sequence in ( + [["alt", "tab"]], [["alt", "f4"]], [["ctrl", "o"]], [["ctrl", "l"]])) + if self.control is None or pending.get("control_scope") != self.control["scope_key"]: + return + current = _content_state(targets) + scenes = self.scenes.setdefault(self.control["scope_key"], [pending["control_state"]]) + revisited = current in scenes + scenes.append(current) + del scenes[:-16] + local = not returncode and effect != "not_met" and ( + effect == "met" or current != pending["control_state"]) + self.local_effect = "confirmed_input" if effect == "met" else ( + "controls_changed" if local else "unknown" if changed else "no_observed_effect") + if self.memory is not None: + blocker = self.memory.active_blocker + if (blocker and blocker["owner"] == self.control["scope_key"] + and blocker["kind"] in {"rejected_completion", "no_effect"} + and (effect == "met" or local and not revisited + or pending.get("visual_action") and changed)): + self.memory.resolve_blocker(blocker["owner"], evidence_ref) + self.control.pop("completion_rejected", None) + elif not local and (self.control["without_effect"] or returncode): + self.memory.block("no_effect", self.control["scope_key"], + "The last interaction did not advance the delegated objective", evidence_ref, + attempt={"action": pending["operation"], "result": "no_effect"}) + if not returncode and effect == "met": + self.control.update(attempts={}, without_effect=0, visited_focus=[], ground_calls=0, + ground_seconds=0.0) + self.checkpoint_due = False + return + if local and not revisited: + if not self.control["recovery_required"]: + self.control.update(ground_calls=0, ground_seconds=0.0) + self.checkpoint_due = False + return # New content permits exploration, but does not release previously failed paths. + self.control["without_effect"] += 1 + sequence = pending["action"].get("parameters", {}).get("sequence") + family = pending.get("execution_family") or _navigation_family(pending["operation"], sequence) + if family and not pending.get("commit_action"): + attempts = self.control["attempts"] + attempts[family] = attempts.get(family, 0) + 1 + focused = [t.semantic_key or t.name for t in targets if t.focused] + focus = "|".join(focused) + visited = self.control["visited_focus"] + cycle = bool(focus and focus in visited) + if family == "focus_traversal" and focus and focus not in visited: + visited.append(focus) + self.checkpoint_due = False # Visiting another known field is finite exploration. + else: + self.checkpoint_due = attempts[family] >= (3 if family.startswith("target:") else 4) and ( + family != "focus_traversal" or not focus or cycle) + elif self.control["without_effect"] >= 8: + self.checkpoint_due = True + + def reusable_visual(self, frame, state, screenshot): + if not frame: + return False + return (not self.invalidate_visual and frame["state"] == state + and not visual_changed(frame["screenshot"], screenshot)) + + + def finalize(self, space, targets, stage, memory, tree, observation_ref=""): + """All producers run before the final scope/failed-path checks. No action executes here.""" + self.memory, self.observation_ref = memory, observation_ref + def scope(): + result = modal_frontier(space, targets, tree) + if result: + return result # Resolve the actual input owner before recovering an underlying document. + if stage.get("document_recovery") or stage.get("application_recovery"): + result.update(document_frontier(space, targets, stage.get("target_document"), + stage.get("target_application"))) + return result + + scope() + interaction = self.constrain(space, targets, stage, memory, tree=tree) + limits = scope() # Commit candidates must obey the same scope as every other producer. + root = ET.fromstring(tree or "") + overlay = limits.get("blocking_overlay", "") + is_dialog = "blocking_overlay" in limits + bounds = limits.get("overlay_bounds") + owner = portal_needs_grounding(tree) or unrepresented_window(root) or ( + "dialog@" + str([round(v / 32) for v in bounds]) if bounds else overlay) + key = json.dumps([root.get("foreground_class", ""), root.get("foreground_title", ""), + owner], ensure_ascii=False) + blocker = memory.active_blocker + if (blocker and blocker["status"] != "resolved" and blocker["kind"] == "input_obstructed" + and not is_dialog and json.loads(blocker["owner"])[0] == root.get("foreground_class", "")): + memory.resolve_blocker(blocker["owner"], observation_ref) + for old in memory.execution_constraints: + if old.get("scope_key") == blocker["owner"]: + old.update(recovery_required=False, blocked_families=[], + ground_calls=0, ground_seconds=0.0) + memory.work_items[blocker["work_item_id"]]["recovery_required"] = False + control = next((c for c in memory.execution_constraints + if c.get("kind") == "execution_checkpoint" and c["scope_key"] == key), None) + if control is None: + control = {"kind": "execution_checkpoint", "scope_key": key, + "work_item_id": stage["work_item_id"], "global_scope": bool(owner), + "attempts": {}, "blocked_families": [], "without_effect": 0, + "visited_focus": [], "failed_stages": 0, "recovery_required": False} + memory.execution_constraints.append(control) + self.control, self.content_state = control, _content_state(targets) + by_id = {t.id: t for t in targets} + for operation, candidates in space.target_heads.items(): + for candidate in candidates: + target = by_id.get(candidate.target_id) + identity = target.semantic_key if target else candidate.metadata.get("target_label", "") + candidate.metadata["execution_family"] = _navigation_family( + candidate.operation, _keys(candidate)) + if candidate.operation in {"click", "double_click"} and identity: + candidate.metadata["execution_family"] = "target:" + identity.casefold() + space.target_heads[operation] = [c for c in candidates if c.metadata.get("commit_action") + or c.metadata.get("execution_family") not in control["blocked_families"]] + active = targets + if bounds := limits.get("overlay_bounds"): + x, y, width, height = bounds + active = [t for t in targets if None not in (t.x, t.y, t.width, t.height) + and x <= t.x + t.width // 2 < x + width and y <= t.y + t.height // 2 < y + height] + elif limits: + ids = {c.target_id for c in space.candidates() if c.target_id} + active = [t for t in targets if t.id in ids] + active_inputs = [v for v in stage["inputs"] + if self.fields.get(_binding(v), {}).get("phase") != "consumed"] + frontier = input_frontier(space, active, active_inputs) + typing = next((c for c in space.candidates() if c.operation == "type_text"), None) + specific = next((c.description for c in space.candidates() if c.metadata.get("commit_action")), + typing.description + f" Supplied text: {typing.metadata['text']!r}." if typing + else frontier["next_intent"]) + intent = specific or interaction.get("next_intent", "") or stage["goal"] + if is_dialog: + intent = f"Within the active {overlay!r} dialog: " + intent + requested = any(c.source == "goal_input" or c.metadata.get("commit_action") + for c in space.candidates()) or any( + not launcher_input(v) and matches_field(v["text"], node.get("name", "")) + for v in stage["inputs"] for node in root.iter()) + recovery = bool(is_dialog and limits.get("overlay_kind") != "task_dialog" and not requested + and (limits.get("overlay_kind") == "obstruction" or control["recovery_required"])) + if recovery: + memory.block("input_obstructed", key, overlay, observation_ref) + intent = (f"Restore input access blocked by the active {overlay!r} dialog. Use its visible " + "dismissal controls; do not edit the background or repeat failed shortcuts.") + if limits and not is_dialog: + intent = limits["next_intent"] + (" Next legal input: " + specific if specific else "") + if control.get("completion_rejected"): + if self.rejected_states.get(key) == self.content_state: + space.controls.pop("done", None) + else: + control.pop("completion_rejected", None) + memory.resolve_blocker(key, observation_ref) + if frontier["unbound_inputs"] and not any(c.source == "goal_input" for c in space.candidates()): + memory.block("target_unbound", key, ", ".join(frontier["unbound_inputs"]), observation_ref) + return {**interaction, **frontier, **limits, "next_intent": intent, + "scope_recovery": recovery, + "active_blocker": deepcopy(memory.active_blocker), + "execution_control": deepcopy(control)} + + def choice_context(self, context): + """One current objective, not the original planner's speculative navigation script.""" + scoped = "blocking_overlay" in context or context.get("document_recovery") or context.get( + "application_recovery") + recovery = context.get("scope_recovery", False) + restoring = recovery or context.get("document_recovery") or context.get("application_recovery") + return {"objective": context.get("next_intent") if restoring else context["goal"], + "delegated_goal": context["goal"], "next_action_hint": context.get("next_intent", ""), + "method": "" if restoring else context.get("method", ""), + "guidance": "" if restoring else context.get("guidance", ""), + "exit_condition": ("The obstruction is gone; refresh and resume the delegated objective." + if recovery else context["completion_criteria"]), + "input_owner": context.get("blocking_overlay", context.get("target_application")), + "constraints": context.get("constraints", []), + "inputs": [] if recovery else [v for v in context["inputs"] + if v["label"] not in context.get("unbound_inputs", [])] if scoped + else [v for v in context["inputs"] + if self.fields.get(_binding(v), {}).get("phase") != "consumed"], + "unbound_inputs": [] if recovery else context.get("unbound_inputs", []), + "confirmed_inputs": context.get("satisfied_inputs", []), + "failed_paths": context.get("execution_control", {}).get("blocked_families", []), + "failures": [{k: f[k] for k in ("error", "count") if k in f} + for f in context.get("failures", [])[-2:]], + "active_blocker": context.get("active_blocker"), + "grounding_exhausted": context.get("grounding_exhausted", False), + "scope_rule": ("Interact only with the active input owner. A business dialog may be " + "the task itself; do not dismiss it just because it is a dialog." + if scoped else "Use the current observation, not historical UI state."), + "actions_remaining": context.get("actions_remaining")} + + def finish(self, memory, result, history): + """Only a resolved execution condition releases a failed path; unrelated facts cannot.""" + stage = memory.active_stage + if stage["route"] != "gui" or self.control is None: + return + control = self.control + if not history and result["stage_outcome"] != "met": + control["completion_rejected"] = True + self.rejected_states[control["scope_key"]] = self.content_state + memory.block("rejected_completion", control["scope_key"], result["summary"], + next(iter(result["evidence_refs"]), ""), + attempt={"action": "done", "result": result["stage_outcome"]}) + elif not history and result["stage_outcome"] == "met": + return # A genuinely satisfied initial state need not be modified. + needed = {f["key"] for f in stage.get("fact_targets", [])} & memory.source_keys + current_refs = {h["evidence_ref"] for h in history} + source_complete = (stage["purpose"] == "prepare" and not control["global_scope"] + and result["stage_outcome"] == "met" + and any(f["key"] in needed and f["status"] == "confirmed" + and set(f["evidence_refs"]) & current_refs for f in result["facts"])) + resolved = result["stage_outcome"] == "met" and result.get("progress_kind") in { + "navigation", "content", "save", "inspection"} + if source_complete or result["unit_met"] or resolved: + control.update(attempts={}, without_effect=0, visited_focus=[], failed_stages=0, + ground_calls=0, ground_seconds=0.0, blocked_families=[], recovery_required=False) + else: + blocked = [family for family, count in control["attempts"].items() + if count >= (3 if family.startswith("target:") else 4)] + control["failed_stages"] += 1 + if blocked or control["failed_stages"] >= 2: + control["blocked_families"] = sorted(set(control["blocked_families"] + blocked)) + control["recovery_required"] = True + control["last_evidence_ref"] = (history[-1]["evidence_ref"] if history + else next(iter(result["evidence_refs"]), "")) + memory.work_items[stage["work_item_id"]]["recovery_required"] = True + if not control["recovery_required"]: + memory.work_items[stage["work_item_id"]]["recovery_required"] = False + self.checkpoint_due = False + + def constrain(self, space, targets, stage, memory, tree=""): + if self.stage_id != stage["id"]: + self.stage_id, self.fields = stage["id"], {} + self.invalidate_visual = False + by_id = {t.id: t for t in targets} + application = stage.get("target_application") + for operation, candidates in space.target_heads.items(): + space.target_heads[operation] = [c for c in candidates if not ( + application and (t := by_id.get(c.target_id)) and t.application == "gnome-shell" + and t.role in {"push_button", "button"} and t.name.casefold() != "files" + and application_mismatch(application, + ET.tostring(ET.Element("desktop-frame", foreground_class=t.name), encoding="unicode"), + known_only=True))] + available = {} + for candidate in space.candidates(): + if candidate.operation != "type_text": + continue + for value in stage["inputs"]: + if (candidate.metadata.get("text", "").rstrip() != value["text"].rstrip() + or launcher_input(value)): + continue + if not matches_field(value.get("target_hint", value["label"]), + candidate.metadata.get("target_hint", "")): + continue + key = _binding(value) + field = self.fields.setdefault(key, { + "target": value.get("target_hint", value["label"]), + "expected_value": value["text"].rstrip(), + "phase": "needs_input", "commit": "enter" if _command_input(value) else None}) + target = by_id.get(candidate.target_id) + point = ([target.x + target.width // 2, target.y + target.height // 2] if target + else candidate.metadata.get("visual_point")) + available[key] = point + candidate.metadata["interaction_binding"] = key + if field["commit"]: + candidate.metadata["text"] = candidate.metadata["text"].rstrip("\r\n") + # Satisfied semantic fields need no typing candidate but may still need an explicit commit. + for value in stage["inputs"]: + key = _binding(value) + target = _input_target(value, targets) + field = self.fields.get(key) + title = ET.fromstring(tree or "").get("foreground_title", "") + if (field and value["text"].startswith("/") and field["phase"] in { + "needs_commit", "submitted_unconfirmed", "typed_unconfirmed"} + and Path(value["text"]).name.casefold() in title.casefold()): + # The opened document is evidence; a vanished path box is expected. + field["phase"] = "consumed" + if key in self.fields and target is not None: + available[key] = [target.x + target.width // 2, target.y + target.height // 2] + if input_satisfied(value, target) and not self.fields[key]["commit"]: + self.fields[key]["phase"] = "confirmed" + commits = [] + for key, field in self.fields.items(): + if field["phase"] == "needs_commit" and available.get(key): + commits.append(ActionCandidate( + id="COMMIT:" + key, operation="keypress", source="interaction", + description=f"Focus {field['target']!r} and press ENTER to submit the previously typed " + "command. Its result still needs observation; do not request completion yet.", + metadata={"key_sequence": [["enter"]], "focus_point": available[key], + "interaction_binding": key, "commit_action": True})) + for operation, candidates in space.target_heads.items(): + space.target_heads[operation] = [c for c in candidates if not ( + (key := c.metadata.get("interaction_binding")) in self.fields + and self.fields[key]["phase"] != "needs_input")] + if commits: + space.target_heads["keypress"] = [c for c in space.target_heads["keypress"] + if _keys(c) != [["enter"]]] + commits + pending = [c for c in space.candidates() if c.operation == "type_text"] + for operation, candidates in space.target_heads.items(): + space.target_heads[operation] = [c for c in candidates if not ( + c.operation == "type_text" and memory.blocked_command( + c.metadata.get("text", ""), application))] + pending = [c for c in pending if not memory.blocked_command(c.metadata.get("text", ""), application)] + if pending or commits: + space.controls.pop("done", None) + uncertain = [v["target"] for v in self.fields.values() if v["phase"] == "typed_unconfirmed"] + label = (pending[0].metadata.get("field_label") or pending[0].metadata.get("target_hint", "field") + if pending else "") + typing = (f"Enter supplied {pending[0].metadata['text']!r} into the observed " + f"{label!r}." + if pending else "") + if pending and pending[0].metadata.get("choice_input"): + typing = pending[0].description + intent = (commits[0].description if commits else typing if pending else + "Input dispatched but value unconfirmed for " + ", ".join(uncertain) + + ". Inspect the visible result and continue: " + stage["goal"] if uncertain else "") + return {"interaction": self.snapshot(), "next_intent": intent} if intent else { + "interaction": self.snapshot()} + + def snapshot(self): + return {"stage_id": self.stage_id, "fields": deepcopy(list(self.fields.values())), + "execution_control": deepcopy(self.control), "local_effect": self.local_effect} diff --git a/src/tandem/gui/semantics.py b/src/tandem/gui/semantics.py index e68904d..37d67ac 100644 --- a/src/tandem/gui/semantics.py +++ b/src/tandem/gui/semantics.py @@ -17,7 +17,8 @@ def _norm(value: str) -> str: def _key(target: UITarget) -> str: - return "|".join((target.role, _norm(target.name), *(_norm(p) for p in target.parent_context))) + return "|".join((target.role, _norm(target.name), *(_norm(p) for p in target.parent_context), + _norm(target.nearby_text) if "dom" in target.source else "")) def _bounds(element: ET.Element) -> tuple[int, int, int, int] | None: @@ -38,11 +39,13 @@ def active(element: ET.Element) -> bool: for key, value in element.attrib.items() ) - for app in root: + apps = [root] if root.tag.rsplit("}", 1)[-1] == "application" else list(root) + for app in apps: app_active = active(app) or any(active(child) for child in app) chrome_active = chrome_active or ( app_active and "chrome" in app.attrib.get("name", "").lower() ) + owners = {element: app.get("name", "") for app in apps for element in app.iter()} for element in root.iter(): role = element.tag.rsplit("}", 1)[-1].replace("-", "_") if role not in _LABEL_ROLES: @@ -51,7 +54,8 @@ def active(element: ET.Element) -> bool: text = " ".join((element.attrib.get("name") or element.text or "").split()) if box and text and len(text) <= 120: x, y, width, height = box - labels.append({"text": text, "box": box, "cx": x + width / 2, "cy": y + height / 2}) + labels.append({"text": text, "box": box, "cx": x + width / 2, "cy": y + height / 2, + "application": owners.get(element, "")}) return labels, chrome_active @@ -61,9 +65,13 @@ def _nearest_label(target: UITarget, labels: list[dict]) -> str: cx, cy = target.x + target.width / 2, target.y + target.height / 2 best, best_score = "", float("inf") for item in labels: + if item["application"] != target.application: + continue x, y, width, height = item["box"] - left = x + width <= target.x + 4 and abs(item["cy"] - cy) <= max(14, target.height) - above = y + height <= target.y + 4 and x + width >= target.x and x <= target.x + target.width + left = (0 <= target.x - x - width <= 160 + and abs(item["cy"] - cy) <= max(14, min(40, target.height))) + above = (0 <= target.y - y - height <= 48 and x + width >= target.x + and x <= target.x + target.width) if not left and not above: continue score = ( @@ -137,7 +145,7 @@ def _dom_target(item: dict, snapshot_id: str) -> UITarget | None: showing=True, selected=item.get("selected"), checked=item.get("checked"), - focused=None, + focused=item.get("focused"), x=x, y=y, width=width, @@ -151,6 +159,8 @@ def _dom_target(item: dict, snapshot_id: str) -> UITarget | None: scope="web_content", label_source=str(item.get("label_source") or "dom"), operations=tuple(item.get("operations") or ("click",)), + input_type=str(item.get("input_type") or ""), + date_order=tuple(item.get("date_order") or ()), ) return replace(target, semantic_key=_key(target)) @@ -164,7 +174,7 @@ def _merge(base: UITarget, dom: UITarget) -> UITarget: merged = replace( base, name=dom.name if use_dom_name else base.name, - value=dom.value or base.value, + value=dom.value if "type_text" in dom.operations else dom.value or base.value, checked=dom.checked if dom.checked is not None else base.checked, selected=dom.selected if dom.selected is not None else base.selected, source="a11y+dom", @@ -173,10 +183,50 @@ def _merge(base: UITarget, dom: UITarget) -> UITarget: scope="web_content", label_source=dom.label_source, operations=dom.operations, + input_type=dom.input_type or base.input_type, + date_order=dom.date_order or base.date_order, ) return replace(merged, semantic_key=_key(merged)) +def _date_fields(targets): + """Compose adjacent, explicitly named date segments; never infer order from locale or a screenshot.""" + parts = [replace(t, name=t.name.split(" ", 1)[0], + nearby_text=t.nearby_text or t.name.partition(" ")[2]) + for t in targets if t.role == "spin_button" + and t.name.casefold().split(" ", 1)[0] in {"day", "month", "year"}] + for day in [t for t in parts if t.name.casefold() == "day"]: + nearby = [t for t in parts if t.application == day.application + and t.parent_context == day.parent_context and abs(t.y - day.y) <= 4 + and abs(t.x - day.x) <= 180] + if len(nearby) != 3 or {t.name.casefold() for t in nearby} != {"day", "month", "year"}: + continue + name = next((t.nearby_text for t in nearby if re.search(r"[\w]{3}", t.nearby_text)), "") + if not name: + continue + ordered = sorted(nearby, key=lambda t: t.x) + values = {t.name.casefold(): t.value for t in nearby} + value = (f"{int(values['year']):04d}-{int(values['month']):02d}-{int(values['day']):02d}" + if all(v.isdigit() and int(v) > 0 for v in values.values()) else "") + left, top = min(t.x for t in nearby), min(t.y for t in nearby) + field = replace(day, id="date:" + day.id, role="entry", name=name, value=value, + x=left, y=top, width=max(t.x + t.width for t in nearby) - left, + height=max(t.y + t.height for t in nearby) - top, description="Segmented date", + operations=("click", "type_text"), input_type="date", + date_order=tuple(t.name.casefold() for t in ordered)) + yield replace(field, semantic_key=_key(field)) + + +def unrepresented_window(root: ET.Element | None) -> str: + """A foreground window absent from A11y must not inherit the background page's controls.""" + if root is None: + return "" + title = root.get("foreground_title", "") + windows = {n.get("name") for n in root.iter() + if n.tag.rsplit("}", 1)[-1] in {"frame", "dialog"} and n.get("name")} + return title if windows and title not in windows else "" + + def enrich_targets( targets: list[UITarget], accessibility_tree: str | None, @@ -188,6 +238,8 @@ def enrich_targets( root = ET.fromstring(accessibility_tree) if accessibility_tree else None except ET.ParseError: root = None + if unrepresented_window(root): + return [] # The existing visual fallback must locate this real input owner first. labels, chrome_active = _a11y_context(root) content_y = _content_y(targets, browser_snapshot) enriched = [] @@ -196,7 +248,9 @@ def enrich_targets( if target.role == "page_tab": name = _MEMORY_USAGE.sub("", name) nearby = target.nearby_text or _nearest_label(target, labels) - if not name and nearby: + if not name and nearby and (target.role in {"entry", "text", "textbox", "searchbox", + "spin_button", "combo_box"} + or "type_text" in target.operations): name, source = nearby, "a11y_nearby" if not name and target.focused: name, source = f"Focused {target.role.replace('_', ' ')}", "synthetic" @@ -210,6 +264,7 @@ def enrich_targets( label_source=source, ) enriched.append(replace(item, semantic_key=_key(item))) + enriched.extend(list(_date_fields(enriched))) chrome_targets = any( "chrome" in target.application.lower() or "chromium" in target.application.lower() for target in targets diff --git a/src/tandem/gui/targets.py b/src/tandem/gui/targets.py index 0f7bd8a..5e9ba24 100644 --- a/src/tandem/gui/targets.py +++ b/src/tandem/gui/targets.py @@ -33,6 +33,10 @@ class UITarget: scope: str = "native" label_source: str = "a11y" operations: tuple[str, ...] = () + range_min: float | None = None + range_max: float | None = None + input_type: str = "" + date_order: tuple[str, ...] = () def compact(self) -> dict: return {key: value for key, value in asdict(self).items() if value not in (None, "", (), False)} @@ -100,6 +104,32 @@ def _attribute(element: ET.Element, name: str) -> str: return "" +def _number(element, name): + value = _attribute(element, name) + return float(value) if re.fullmatch(r"-?\d+(?:\.\d+)?", value) else None + + +def _is_overlay(element): + return _local(element.tag) in {"dialog", "alert", "popup_menu"} or ( + _local(element.tag) == "menu" and _attribute(element, "ispopup") in {"true", "auto", "menu"}) + + +def _field_labels(root): + labels = {} + for parent in root.iter(): + preceding = "" + for child in parent: + if _local(child.tag) in {"label", "static"}: + preceding = str(child.get("name") or child.text or "").strip() + continue + fields = [e for e in child.iter() if _state(e, "editable") is True + or _local(e.tag) in {"entry", "textbox", "searchbox", "combo_box", "spin_button"}] + if preceding and len(fields) == 1: + labels[fields[0]] = preceding + preceding = "" + return labels + + class A11yTargetBuilder: def build( self, @@ -108,23 +138,43 @@ def build( max_targets: int = 64, infer_button_labels: bool = False, include_sliders: bool = False, + context: str = "", + filter_occluded: bool = False, ) -> list[UITarget]: root = ET.fromstring(raw_tree) targets: list[UITarget] = [] + field_labels = _field_labels(root) if filter_occluded else {} + target_nodes, overlays = {}, {} + if filter_occluded: + for node in root.iter(): + if (_is_overlay(node) and _state(node, "showing") is True + and _state(node, "visible") is not False): + position, size = _pair(_attribute(node, "screencoord")), _pair(_attribute(node, "size")) + if position and size and min(size) > 0: + overlays.setdefault((*position, *size), set()).update(node.iter()) - def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name: str) -> None: + def visit(element, active, parents, app_name, modal=False, roles=(), file_manager=False): active = active or _state(element, "active") is True role = _local(element.tag) + modal = modal or _is_overlay(element) raw_name = str(element.attrib.get("name") or "").strip() name = raw_name if role == "application": app_name = name label_source = "a11y" if name else "unresolved" + if not name and element in field_labels: + name, label_source = field_labels[element], "a11y_adjacent" + if (not name and filter_occluded and "gnome-shell" in parents + and "Overview" in parents and _state(element, "editable") is True): + name, label_source = "Desktop search", "a11y_context" + file_manager = file_manager or "nautilus" in app_name.lower() or "file chooser" in name.lower() value = ( str(element.text or "").strip() if role in {"text", "textbox", "entry", "searchbox", "spin_button"} - else "" + else _attribute(element, "value") if role == "slider" else "" ) + if role == "spin_button" and not value and (number := _number(element, "value")) is not None: + value = format(number, "g") position = _pair(_attribute(element, "screencoord")) size = _pair(_attribute(element, "size")) if infer_button_labels and not name and role in {"button", "push_button"} and position and size: @@ -149,6 +199,9 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name: and active and ( any("file chooser" in item.lower() or item.lower() == "places" for item in parents) + or filter_occluded and file_manager and any(r in { + "list_item", "tree_item", "button", "push_button", "menu_item", "option", "link"} + for r in roles) or bool( re.search( r"\b(?:monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b", name, re.I @@ -158,7 +211,8 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name: ) if ( (active or system_target) - and (role in SUPPORTED_ROLES or safe_label or (include_sliders and role == "slider")) + and (role in SUPPORTED_ROLES or safe_label or (include_sliders and role == "slider") + or filter_occluded and (role == "image" or file_manager and role == "canvas")) and (name or role in SUPPORTED_ROLES) and interactive_text and enabled is not False @@ -168,7 +222,6 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name: and size and size[0] > 0 and size[1] > 0 - and len(targets) < max_targets ): normalized_name = " ".join(name.lower().split()) parent_context = tuple(item for item in parents[-3:] if item) @@ -198,10 +251,24 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name: or _attribute(element, "placeholder-text") ), label_source=label_source, + source="a11y_dialog" if filter_occluded and modal else "a11y", + range_min=_number(element, "min") if role == "slider" else None, + range_max=_number(element, "max") if role == "slider" else None, ) ) + target_nodes[targets[-1].id] = element for child in element: - visit(child, active, context[-4:], app_name) + visit(child, active, context[-4:], app_name, modal, (*roles, role), file_manager) visit(root, False, (), "") - return targets + if filter_occluded: + targets = [t for t in targets if not any( + target_nodes[t.id] not in nodes and x <= t.x + t.width // 2 < x + width + and y <= t.y + t.height // 2 < y + height + for (x, y, width, height), nodes in overlays.items())] + if context: + words = set(re.findall(r"\w+", context.lower())) + targets.sort(key=lambda t: (t.source == "a11y_dialog", + len(words & set(re.findall(r"\w+", t.name.lower()))), t.focused is True, + t.role in {"button", "push_button", "combo_box", "entry", "textbox"}), reverse=True) + return targets[:max_targets] diff --git a/src/tandem/memory/__init__.py b/src/tandem/memory/__init__.py new file mode 100644 index 0000000..e6c80da --- /dev/null +++ b/src/tandem/memory/__init__.py @@ -0,0 +1 @@ +"""Task-scoped source facts, delivery progress and execution evidence.""" diff --git a/src/tandem/memory/context.py b/src/tandem/memory/context.py new file mode 100644 index 0000000..33d304f --- /dev/null +++ b/src/tandem/memory/context.py @@ -0,0 +1,83 @@ +"""Compact role-specific views. Raw receipts/screenshots remain in the episode record.""" + +import re + + +def _fact_view(fact, limit=2400): + view = {k: v for k, v in fact.items() if k not in {"observations", "entries"}} + if len(view["value"]) > limit: + view["value"] = (view["value"][:limit // 2] + + "\n[excerpt; request read_memory for the stored content]\n" + + view["value"][-limit // 2:]) + view["value_excerpt_only"] = True + return view + + +def planning_context(memory): + ready = memory.ready_items()[:4] + keys = {f["key"] for item in ready for f in ( + memory.preparation_target(item)[0]["facts_needed"] if item["kind"] == "prepare" + else item["facts_needed"])} + facts = [f for f in memory.facts.values() if f["scope"] == "task"] + auxiliary = {f["key"] for f in facts[-8:]} + visible = [_fact_view(f) for f in facts if f["key"] in keys | auxiliary] + return {"requirements": [{k: r[k] for k in ("id", "requirement", "kind", "status")} + for r in memory.requirements], + "ready_work_items": [{**item, "can_start_partial": item["kind"] == "deliver", + "next_acquisition": { + "unit": memory.preparation_target(item)[0]["goal"], + "missing_facts": memory.preparation_target(item)[1]}} + for item in ready], + "completed_work": [i["id"] for i in memory.work_items.values() if i["status"] == "done"], + "known_facts": visible, + "fact_index": [{**{k: f[k] for k in ("key", "status", "source_ref")}, + "complete": f.get("complete", True)} for f in facts], + "historical_ui_notes": [f for f in memory.facts.values() if f["scope"] == "stage"][-6:], + "materials": [{"source_ref": m["source_ref"], "covered_parts": m["covered_parts"], + "summary_excerpt": m["summary"][:1800], + "summary_excerpt_only": len(m["summary"]) > 1800} + for m in list(memory.materials.values())[-8:]], + "artifacts": list(memory.artifacts.values()), "failures": memory.failures[-12:], + "pending_publications": [a for a in memory.artifacts.values() if a.get("delivery_pending")], + "unsupported_methods": [{**f["observed_method"], "observed_error": f["error"], + "evidence_refs": f["evidence_refs"]} + for f in memory.failures if f.get("observed_method")], + "observed_missing_modules": sorted({name for f in memory.failures for name in + re.findall(r"No module named ['\"]([^'\"]+)['\"]", f["error"])}), + "execution_constraints": memory.execution_constraints[-12:], + "active_blocker": memory.active_blocker, + "memory_update_rejections": memory.last_rejections, + "last_interaction_historical": memory.last_interaction, + "stages_without_progress": memory.stats["stages_without_progress"]} + + +def execution_context(memory): + stage = memory.active_stage + item = memory.work_items[stage["work_item_id"]] + return {"goal": stage["goal"], "completion_criteria": stage["completion_criteria"], + "method": stage["method"], + "constraints": [r["requirement"] for r in memory.requirements if r.get("kind") == "constraint"], + "inputs": stage["inputs"], "guidance": stage["guidance"], + "target_application": stage.get("target_application"), + "facts": [_fact_view(memory.facts[k]) for k in stage["input_fact_keys"] if k in memory.facts], + "delivery_state": item["delivery_state"], "remaining": item["remaining"], + "completed_milestones": item["milestones"], + "failures": [f for f in memory.failures if f["work_item_id"] == item["id"]][-4:], + "execution_constraints": [c for c in memory.execution_constraints + if c["work_item_id"] == item["id"] or c.get("global_scope")][-6:], + "active_blocker": memory.active_blocker, + "actions_remaining": stage["max_actions"] - stage["actions"], + "control": "Choose each next action from the NEW observation. DONE requests stage verification, " + "not whole-task completion. Do not repeat satisfied input or toggle states."} + + +def verification_context(memory): + """Independent stage evidence remains available at whole-task verification.""" + return {"requirements": memory.requirements, + "work_items": [{k: item[k] for k in ("id", "goal", "status", "delivery_state", "remaining", + "evidence_refs", "milestones")} + for item in memory.work_items.values() if item["kind"] == "deliver"], + "source_facts": [_fact_view(f) for f in memory.facts.values() + if f["scope"] == "task" and f["status"] == "confirmed"], + "artifacts": list(memory.artifacts.values()), + "active_blocker": memory.active_blocker} diff --git a/src/tandem/memory/state.py b/src/tandem/memory/state.py new file mode 100644 index 0000000..b66d1af --- /dev/null +++ b/src/tandem/memory/state.py @@ -0,0 +1,355 @@ +"""Host-owned task memory. Plans request work; stage verification commits progress.""" + +import json +import re +from copy import deepcopy +from pathlib import Path + +LEVELS = {"none": 0, "implemented": 1, "saved": 2, "verified": 3} + + +class TaskMemory: + def __init__(self, instruction, requirements): + self.instruction = instruction + self.requirements = deepcopy(requirements) + ids = [r["id"].strip() for r in self.requirements] + if not all(ids) or len(set(ids)) != len(ids): + raise ValueError("Requirement IDs must be nonempty and unique") + self.facts = {} + self.materials = {} + self.work_items = {} + self.artifacts = {} + self.failures = [] + self.execution_constraints = [] + self.active_stage = None + self.stage_count = 0 + self.revision = 0 + self.known_evidence = set() + self.last_rejections = [] + self.last_interaction = {} + self.active_blocker = None + sources = {f["key"]: f for r in self.requirements for f in r["facts_needed"]} + self.source_keys = set(sources) + self.stats = {"stages": 0, "verified_stages": 0, "gui_actions": 0, + "cli_actions": 0, "new_facts": 0, "delivery_advances": 0, + "source_parts": 0, "milestone_advances": 0, "stages_without_progress": 0} + for req in self.requirements: + req.update(status="unknown", evidence_refs=[]) + if req.get("kind") == "constraint": + continue + needed = [sources[key] for key in dict.fromkeys( + key for keys in req["unit_fact_keys"] for key in keys)] + preparation = f"{req['id']}:prepare" + if needed: + self._add(preparation, req, "prepare", + "Acquire missing source facts for a ready unit", [], needed) + for index, unit in enumerate(req["units"], 1): + keys = req["unit_fact_keys"][index - 1] + self._add(f"{req['id']}:unit:{index}", req, "deliver", unit, [], + [sources[key] for key in keys]) + + def _add(self, key, req, kind, goal, dependencies, facts): + if key in self.work_items: + raise ValueError("Duplicate work item: " + key) + self.work_items[key] = { + "id": key, "requirement_ids": [req["id"]], "kind": kind, "goal": goal, + "completion_criteria": goal, "depends_on": dependencies, + "facts_needed": facts, "status": "pending", + "delivery_state": "none", "remaining": [], "evidence_refs": [], + "milestones": {}, "failed_cli_stages": 0, + } + + def _facts_ready(self, item): + return all(self.facts.get(f["key"], {}).get("status") == "confirmed" + and self.facts[f["key"]]["scope"] == "task" + and self.facts[f["key"]].get("complete", True) for f in item["facts_needed"]) + + def preparation_target(self, item): + units = [item] if item["kind"] == "deliver" else [ + w for w in self.work_items.values() if w["kind"] == "deliver" + and w["status"] != "done" and w["requirement_ids"] == item["requirement_ids"]] + choices = [(unit, [f for f in unit["facts_needed"] + if not self._facts_ready({"facts_needed": [f]})]) for unit in units] + choices = [(unit, missing) for unit, missing in choices if missing] + return min(choices, key=lambda pair: len(pair[1])) if choices else (item, []) + + def ready_items(self): + for item in self.work_items.values(): + if item["kind"] == "prepare" and item["status"] != "done" and self._facts_ready(item): + item.update(status="done", evidence_refs=[ref for f in item["facts_needed"] + for ref in self.facts[f["key"]]["evidence_refs"]]) + return sorted([i for i in self.work_items.values() if i["status"] != "done" + and all(self.work_items[d]["status"] == "done" for d in i["depends_on"])], + key=lambda i: (i["kind"] != "deliver", i.get("stalled_stages", 0), + len(self.preparation_target(i)[1]))) + + def start(self, plan, step): + item = self.work_items[plan["work_item_id"]] + if item not in self.ready_items(): + raise ValueError("Stage work item is not ready or has already completed") + self.stage_count += 1 + self.active_stage = {**deepcopy(plan), "id": f"S{self.stage_count}", + "started_step": step, "actions": 0, "status": "running"} + if plan["purpose"] == "prepare": + unit, missing = self.preparation_target(item) + self.active_stage.update(prepares_unit=unit["id"], fact_targets=deepcopy(missing)) + item["status"] = "active" + self.stats["stages"] += 1 + return self.active_stage + + def record_action(self, record): + self.active_stage["actions"] += 1 + self.stats["cli_actions" if record["kind"] == "cli" else "gui_actions"] += 1 + if record["returncode"] != 0: + error = (record.get("stderr") or record.get("stdout") or "Execution failed")[-700:] + self._failure(error, [record["evidence_ref"]]) + + def _failure(self, error, refs, **details): + stage = self.active_stage + matches = [f for f in self.failures if f["work_item_id"] == stage["work_item_id"] + and f["error"] == error] + if matches: + matches[-1]["count"] += 1 + matches[-1]["evidence_refs"] = refs + matches[-1].update(details) + else: + self.failures.append({"stage_id": stage["id"], "work_item_id": stage["work_item_id"], + "method": stage["method"], "error": error, + "count": 1, "evidence_refs": refs, **details}) + + def blocked_command(self, text, application=None): + """Only an observed unsupported command is blocked; advice is never a capability fact.""" + command = re.match(r"\s*([A-Za-z]\w*)\s*\(", text) + if not command: + return "" + for failure in self.failures: + method = failure.get("observed_method", {}) + if method.get("command_name", "").casefold() != command[1].casefold(): + continue + known = method.get("application", "").casefold() + words = [w for w in re.findall(r"[a-z]+", (application or "").casefold()) + if len(w) >= 3 and w not in {"classic", "presentation", "office", "desktop"}] + if application and known and not any(w in known for w in words): + continue + return f"Observed unsupported command {command[1]!r}: {failure['error']}. Use another GUI method." + return "" + + def block(self, kind, owner, subject, evidence_ref, *, attempt=None): + """Execution owns blockers; renaming a plan or switching units cannot clear one.""" + previous = self.active_blocker + if not previous or previous["owner"] != owner or previous["status"] == "resolved": + self.active_blocker = {"kind": kind, "owner": owner, "subject": subject, + "status": "blocked", "resume_stage_id": self.active_stage["id"], + "work_item_id": self.active_stage["work_item_id"], "evidence_refs": [], + "attempts": [], "resolution_evidence_refs": []} + blocker = self.active_blocker + if kind == "input_obstructed" or blocker["kind"] != "input_obstructed": + blocker.update(kind=kind, subject=subject) + if evidence_ref and evidence_ref not in blocker["evidence_refs"]: + blocker["evidence_refs"].append(evidence_ref) + blocker["evidence_refs"] = blocker["evidence_refs"][-4:] + if attempt: + blocker["attempts"].append({**attempt, "evidence_ref": evidence_ref}) + blocker["attempts"] = blocker["attempts"][-8:] + return blocker + + def resolve_blocker(self, owner, evidence_ref): + """Require a new observed resolution, never a planner's promise of recovery.""" + blocker = self.active_blocker + if blocker and blocker["owner"] == owner and evidence_ref: + blocker.update(status="resolved", resolution_evidence_refs=[evidence_ref]) + + def apply(self, result, evidence): + """Only the fixed active unit is writable; references must belong to real evidence.""" + stage = self.active_stage + item = self.work_items[stage["work_item_id"]] + current = set(evidence) + explicit_outputs = {r.get("output_path") for r in self.requirements} + unpublished = [a for a in result["artifacts"] if a["target_path"] != a["working_path"] + or Path(a["target_path"]).is_relative_to("/tmp") + and a["target_path"] not in explicit_outputs] + pending_copy = any(a.get("delivery_pending") and a.get("work_item_id") == item["id"] + for a in self.artifacts.values()) + if (stage["purpose"] != "prepare" and result["delivery_state"] in {"saved", "verified"} + and (unpublished and len(unpublished) == len(result["artifacts"]) + or pending_copy and not result["artifacts"])): + remaining = "Publish the working copy to the user's deliverable and reopen that final file." + result.update(delivery_state="implemented", unit_met=False, + remaining=list(dict.fromkeys(result["remaining"] + [remaining]))) + if "unit_assessment" in result: + result["unit_assessment"].update(status="not_met", remaining=result["remaining"]) + for artifact in unpublished: + artifact["delivery_pending"] = True + self.known_evidence.update(current) + allowed = self.known_evidence + self.last_rejections = [] + advanced = False + for fact in result["facts"]: + fact = deepcopy(fact) + refs = fact["evidence_refs"] + if not refs or not set(refs) <= allowed: + self.last_rejections.append({"key": fact["key"], "reason": "unregistered_evidence"}) + continue + if fact["key"] in self.source_keys and fact["scope"] != "task": + self.last_rejections.append({"key": fact["key"], "reason": "source_key_requires_source_fact"}) + continue + old = self.facts.get(fact["key"]) + if (fact["key"] in self.source_keys and stage["purpose"] != "prepare" + and (old is None or old["value"] != fact["value"])): + self.last_rejections.append({"key": fact["key"], + "reason": "source_update_requires_acquisition_stage"}) + continue + entries = fact.get("entries", {}) + old_entries = old.get("entries", {}) if old else {} + conflicts = any(key in old_entries and old_entries[key] != value + for key, value in entries.items()) + extending = bool(old_entries and entries and not conflicts) + correction = bool(fact.get("correction_reason") and set(refs) & current) + # Incomplete prose sources may arrive a section at a time without structured entries. + # Append evidence, never replace the previously observed section or unlock a partial fact. + if (old and old["status"] == fact["status"] == "confirmed" + and old["scope"] == fact["scope"] == "task" and not old.get("complete", True) + and not entries and not old_entries and set(refs) & current and not correction): + sections = deepcopy(old.get("observations", [{k: old[k] for k in + ("value", "source_ref", "evidence_refs")}])) + if fact["value"] not in [section["value"] for section in sections]: + sections.append({k: fact[k] for k in ("value", "source_ref", "evidence_refs")}) + fact["observations"] = sections + fact["value"] = "\n".join(section["value"] for section in sections) + fact["evidence_refs"] = list(dict.fromkeys(old["evidence_refs"] + refs)) + extending = True + if entries: + fact["entries"] = {**old_entries, **entries} + fact["value"] = json.dumps(fact["entries"], ensure_ascii=False) + fact["evidence_refs"] = list(dict.fromkeys((old or {}).get("evidence_refs", []) + refs)) + fact["complete"] = bool(fact.get("complete", False) or (old or {}).get("complete", False)) + if (old and old["status"] == "confirmed" and old["scope"] == "task" + and (fact["status"] != "confirmed" or fact["scope"] != "task" + or old.get("complete", True) and not fact.get("complete", True) + or old["value"] != fact["value"] and not (correction or extending) + or conflicts and not correction)): + self.last_rejections.append({"key": fact["key"], "reason": "confirmed_source_preserved"}) + continue + if (fact["status"] == "confirmed" and fact["scope"] == "task" + and (old is None or old["value"] != fact["value"] or old["status"] != fact["status"] + or old.get("complete", True) != fact.get("complete", True))): + advanced = advanced or fact["key"] in self.source_keys + self.stats["new_facts"] += 1 + self.facts[fact["key"]] = {**fact, "requirement_ids": item["requirement_ids"], + "observed_stage": stage["id"]} + for material in result["materials"]: + if material["evidence_refs"] and set(material["evidence_refs"]) <= allowed: + old = self.materials.get(material["source_ref"], {}) + new_parts = set(material["covered_parts"]) - set(old.get("covered_parts", [])) + if new_parts: + self.stats["source_parts"] += len(new_parts) + summary = old.get("summary", "") + if material["summary"] not in summary: + summary = (summary + "\n" + material["summary"]).strip() + self.materials[material["source_ref"]] = { + **material, "summary": summary, + "covered_parts": list(dict.fromkeys(old.get("covered_parts", []) + + material["covered_parts"])), + "evidence_refs": list(dict.fromkeys(old.get("evidence_refs", []) + + material["evidence_refs"]))} + refs = result["evidence_refs"] + grounded = bool(set(refs) & current) and set(refs) <= allowed + level = result["delivery_state"] + if result.get("progress_kind") in {"navigation", "source", "none"}: + level = "none" + delivering = item["kind"] == "deliver" and stage["purpose"] != "prepare" + if grounded and delivering and level != "none" and result["stage_outcome"] == "met": + for milestone in result.get("milestones", []): + milestone_refs = milestone["evidence_refs"] + old = item["milestones"].get(milestone["key"]) + if (set(milestone_refs) & current and set(milestone_refs) <= allowed + and (old is None or LEVELS[level] > LEVELS[old["delivery_state"]])): + item["milestones"][milestone["key"]] = {**milestone, "delivery_state": level} + advanced = True + self.stats["milestone_advances"] += 1 + # A verified section is not a verified whole document/application. + if level == "verified" and not result["unit_met"]: + saved = any(a["save_state"] == "saved" for a in result["artifacts"]) + level = "saved" if saved or stage["purpose"] == "save" else "implemented" + if (grounded and delivering and LEVELS[level] > LEVELS[item["delivery_state"]] + and (level != "verified" or result["stage_outcome"] == "met")): + item["delivery_state"] = level + advanced = True + self.stats["delivery_advances"] += 1 + unit_check = result.get("unit_assessment", {}) + remaining = unit_check.get("remaining", result["remaining"]) if unit_check.get( + "work_item_id") == item["id"] else result["remaining"] + item.update(status="pending", remaining=remaining, evidence_refs=refs if grounded else []) + preparation_met = item["kind"] == "prepare" and self._facts_ready(item) + if grounded and result["stage_outcome"] == "met": + self.stats["verified_stages"] += 1 + if preparation_met or (delivering and result["unit_met"] + and level == "verified"): + item["status"] = "done" + advanced = True + for check in result["requirement_checks"]: + if check["evidence_refs"] and set(check["evidence_refs"]) <= allowed: + req = next((r for r in self.requirements if r["id"] == check["id"]), None) + if req is not None: + req.update(status=check["status"], evidence_refs=check["evidence_refs"]) + for artifact in result["artifacts"]: + if not artifact["evidence_refs"] or not set(artifact["evidence_refs"]) <= allowed: + continue + old = self.artifacts.get(artifact["target_path"], {}) + if (old and artifact["working_path"] != old["working_path"] + and (not set(artifact["evidence_refs"]) & current + or artifact.get("replaces_working_path") != old["working_path"])): + self.last_rejections.append({"key": artifact["target_path"], + "reason": "working_copy_change_requires_explicit_migration"}) + continue + self.artifacts[artifact["target_path"]] = { + **artifact, "work_item_id": item["id"], "revision": old.get("revision", 0) + 1} + if artifact["save_state"] == "saved" and artifact["target_path"] == artifact["working_path"]: + for previous in self.artifacts.values(): + if previous["working_path"] == artifact.get("replaces_working_path"): + previous.pop("delivery_pending", None) + for key in result["invalidated_work_item_ids"]: + if key in self.work_items and self.work_items[key]["kind"] == "deliver" and grounded: + self.work_items[key].update(status="pending", delivery_state="none", + milestones={}, + remaining=["Contradictory evidence requires reinspection"]) + for req in self.requirements: + if req["id"] in self.work_items[key]["requirement_ids"]: + req.update(status="uncertain", evidence_refs=list(current)) + for error in result.get("observed_errors", []): + error_refs = set(error["evidence_refs"]) + if not error_refs or not error_refs <= current: + continue + details = {"suggested_actions": result["remaining"]} + name = error.get("command_name", "") + if (name and re.fullmatch(r"[A-Za-z]\w*", name) + and re.search(r"unknown command\s*:\s*" + re.escape(name) + r"\b", + error["message"], re.I)): + details["observed_method"] = {"application": error.get("application", ""), + "command_name": name} + self._failure(error["message"], error["evidence_refs"], **details) + if result["stage_outcome"] != "met" and grounded: + self._failure(result["summary"], refs, suggested_actions=result["remaining"]) + self.stats["stages_without_progress"] = 0 if advanced else self.stats["stages_without_progress"] + 1 + item["stalled_stages"] = 0 if advanced else item.get("stalled_stages", 0) + 1 + item["last_attempt"] = {k: stage[k] for k in ("route", "purpose", "method", "command", "inputs")} + item["last_attempt"]["failure_kind"] = result.get("failure_kind", "") + failed_cli = stage["route"] == "cli" and result.get("failure_kind") == "execution_error" + item["failed_cli_stages"] = item["failed_cli_stages"] + 1 if failed_cli else 0 + self.revision += 1 + self.last_interaction = deepcopy(stage.get("interaction", {})) + self.active_stage = None + self.ready_items() + return advanced + + def snapshot(self): + return deepcopy({"schema_version": 2, "revision": self.revision, "instruction": self.instruction, + "requirements": self.requirements, "materials": list(self.materials.values()), + "facts": list(self.facts.values()), "work_items": list(self.work_items.values()), + "artifacts": list(self.artifacts.values()), "failures": self.failures, + "execution_constraints": self.execution_constraints, + "memory_update_rejections": self.last_rejections, + "last_interaction": self.last_interaction, + "active_blocker": self.active_blocker, + "active_stage": self.active_stage, "stats": self.stats, + "evidence_refs": sorted(self.known_evidence)}) diff --git a/src/tandem/models/jev.py b/src/tandem/models/jev.py index 18950c9..342b2b8 100644 --- a/src/tandem/models/jev.py +++ b/src/tandem/models/jev.py @@ -21,11 +21,12 @@ class JevOmniPolicy: def __init__(self, endpoint: str, timeout_seconds: float = 300, opener=None, - image_stage: tuple[str, str] | None = None, endpoint_provider=None): + image_stage: tuple[str, str] | None = None, endpoint_provider=None, joint_actions=False): self.endpoint = str(endpoint).rstrip("/") self.timeout_seconds = timeout_seconds self.opener = opener or urllib.request.build_opener(urllib.request.ProxyHandler({})) self.endpoint_provider = endpoint_provider + self.joint_actions = joint_actions # (写视角目录, 排序服务宿主机读视角目录):/rank 要求 image_path 在服务端可读, # 截图本体在 adpc 时先暂存到 evobox 中转目录(按内容哈希命名去重)。 self.image_stage = image_stage @@ -164,6 +165,18 @@ def predict(choice_request: dict[str, Any]) -> tuple[list[float], bool]: candidate for candidate in candidates if candidate.operation == selected_operation ] action_request = action_requests[selected_operation] + if self.joint_actions: + # Keep the operation ranking diagnostic; do not prune other concrete goal actions. + descriptions = {choice_id: option for request in action_requests.values() + for choice_id, option in zip( + request["option_ids"], request["options"], strict=True)} + selected_candidates = candidates + action_request = { + "state": operation_request["state"], + "question": "Which complete concrete action should be performed next across ALL operations?", + "option_ids": [c.id for c in candidates], + "options": [f"{c.operation}: {descriptions[c.id]}" for c in candidates], + } if len(selected_candidates) == 1: action_scores, action_used_image = [1.0], operation_used_image else: diff --git a/src/tandem/models/planner_choice.py b/src/tandem/models/planner_choice.py new file mode 100644 index 0000000..b6ffc21 --- /dev/null +++ b/src/tandem/models/planner_choice.py @@ -0,0 +1,56 @@ +"""Choose one offered GUI action through the planner's multimodal API and key pool.""" + +import base64 +import json +import time + +from .ranking import Ranking, build_choice_requests + + +class PlannerChoicePolicy: + def __init__(self, client): + self.client = client + + def rank(self, *, stage_goal, intent, active_window, targets, action_space, + recent_actions, screenshot=None): + candidates = action_space.candidates() + operation, requests = build_choice_requests( + stage_goal=stage_goal, intent=intent, active_window=active_window, targets=targets, + action_space=action_space, recent_actions=recent_actions) + descriptions = {key: text for request in requests.values() + for key, text in zip(request["option_ids"], request["options"], strict=True)} + request = {"state": operation["state"], + "question": "Which complete concrete action should be performed next " + "across ALL operations?", + "option_ids": [c.id for c in candidates], + "options": [f"{c.operation}: {descriptions[c.id]}" for c in candidates]} + content = [{"type": "text", "text": json.dumps(request, ensure_ascii=False)}] + if screenshot is not None: + content.append({"type": "image_url", "image_url": {"url": "data:image/png;base64," + + base64.b64encode(screenshot.read_bytes()).decode("ascii")}}) + messages = [{"role": "system", "content": + "Choose exactly ONE offered action for the current local objective and screenshot. " + "Honor the delegated goal and preservation conditions before next_action_hint; " + "a field hint is not permission to discard an unsaved form or skip a navigation prerequisite. " + "The exit condition is a desired state, NOT an observed fact. Use current evidence; " + "do not repeat rejected completion or ineffective paths. Return only " + '{"choice_id":"exact offered option_id"}. Do not invent actions, coordinates or text.'}, + {"role": "user", "content": content}] + by_id = {c.id: c for c in candidates} + + def validate(value): + okay = isinstance(value.get("choice_id"), str) and value["choice_id"] in by_id + if not okay: + messages.append({"role": "user", "content": + "Invalid choice_id. Return one exact offered ID; no action has been executed."}) + return okay + + started = time.perf_counter() + value = self.client.json_request(messages, validate, role="gui_choice") + winner = by_id[value["choice_id"]] + return Ranking(winner=winner, + action_ranked=[{"id": winner.id, "operation": winner.operation}], + operation_ranked=[], operation_margin=0.0, target_margin=0.0, + latency_ms=round((time.perf_counter() - started) * 1000, 2), + used_image=screenshot is not None, action_request=request, + score_type="api_choice_id") diff --git a/src/tandem/models/ranking.py b/src/tandem/models/ranking.py index 4bb17eb..89ee8ca 100644 --- a/src/tandem/models/ranking.py +++ b/src/tandem/models/ranking.py @@ -22,6 +22,7 @@ class Ranking: confident: bool = True operation_request: dict[str, Any] | None = None action_request: dict[str, Any] | None = None + score_type: str = "fast_model_choice_softmax" def build_choice_requests( @@ -121,7 +122,7 @@ def ranking_dict(value: Ranking) -> dict[str, Any]: "target_margin": value.target_margin, "latency_ms": value.latency_ms, "used_image": value.used_image, - "score_type": "fast_model_choice_softmax", + "score_type": value.score_type, "score_is_calibrated_correctness_probability": False, "operation_request": value.operation_request, "action_request": value.action_request, diff --git a/src/tandem/models/slow.py b/src/tandem/models/slow.py index 35dcaf9..504bdf8 100644 --- a/src/tandem/models/slow.py +++ b/src/tandem/models/slow.py @@ -5,18 +5,86 @@ from __future__ import annotations +import base64 +import io import json import time import urllib.error import urllib.request from pathlib import Path +from PIL import Image + from .key_pool import ApiKeyPool, PoolExhausted, classify_api_error, retry_after_seconds # 空正文(全留在思考通道)或截断时,下一次重试升档思考强度,避免原样重发重复失败 _REASON_EFFORT_LADDER = {"low": "medium", "medium": "high"} +class TaskBudgetExceeded(TimeoutError): + """The agent's task budget ended, not an external API timeout.""" + + +class ModelResponseError(ValueError): + """Bounded repair could not produce a valid structured model response.""" + + +def _pack_images(payload, quality, max_bytes): + """Compress send copies; only exactly identical decoded frames share an image. + + Mutate the existing messages list so validators can still append correction turns on retry. + Never write source files or discard distinct evidence frames to meet the body budget. + """ + frames, pixels = [], [] + stats = {"image_count": 0, "duplicate_images": 0, "source_image_bytes": 0} + for message in payload["messages"]: + if not isinstance(message.get("content"), list): + continue + for part in message["content"]: + if part.get("type") != "image_url": + continue + url = part["image_url"]["url"] + if not url.startswith("data:image/"): + continue + raw = base64.b64decode(url.split(",", 1)[1]) + with Image.open(io.BytesIO(raw)) as source: + frame = source.convert("RGB") + identity = (frame.size, frame.tobytes()) + stats["image_count"] += 1 + stats["source_image_bytes"] += len(raw) + if identity in pixels: + ordinal = pixels.index(identity) + 1 + part.clear() + part.update(type="text", text=f"Same screenshot as supplied image #{ordinal} (exact pixels).") + stats["duplicate_images"] += 1 + else: + pixels.append(identity) + frames.append((part, frame)) + while frames: + image_bytes = 0 + for part, frame in frames: + encoded = io.BytesIO() + frame.save(encoded, format="JPEG", quality=quality, subsampling=0, optimize=True) + raw = encoded.getvalue() + image_bytes += len(raw) + part["image_url"]["url"] = "data:image/jpeg;base64," + base64.b64encode(raw).decode("ascii") + stats.update(image_quality=quality, sent_image_bytes=image_bytes) + if len(json.dumps(payload).encode("utf-8")) <= max_bytes or quality == 65: + break + quality = max(65, quality - 10) + while frames and len(json.dumps(payload).encode("utf-8")) > max_bytes and max( + max(frame.size) for _, frame in frames + ) > 960: + for part, frame in frames: + frame.thumbnail((max(960, int(frame.width * 0.8)), max(540, int(frame.height * 0.8)))) + encoded = io.BytesIO() + frame.save(encoded, format="JPEG", quality=quality, subsampling=0, optimize=True) + part["image_url"]["url"] = "data:image/jpeg;base64," + base64.b64encode( + encoded.getvalue()).decode("ascii") + return {**stats, "unique_images": len(frames), "sent_image_sizes": [list(f.size) for _, f in frames]} + + + def extract_json_object(text: str): """从模型正文中抢救 JSON:剥围栏后,若剩余文本不是纯 JSON,抽取首个平衡的 {…}。 返回解析后的 dict;无 JSON 或解析失败返回 None。""" @@ -75,6 +143,22 @@ def __init__(self, credential_file, config, audit_path: Path, llm_sink=None): # llm_sink(role=..., request=..., response=..., **meta):全量请求/响应落盘钩子, # 由 tandem_eval.recorder 提供(llm_calls.jsonl);None 时仅审计。 self.llm_sink = llm_sink + self.deadline = None + + def set_deadline(self, deadline): + """Absolute monotonic task deadline, shared by requirements, planning and verification.""" + self.deadline = deadline + + def _remaining(self): + remaining = self.deadline - time.perf_counter() if self.deadline is not None else float("inf") + if remaining <= 0: + raise TaskBudgetExceeded("Task deadline exhausted during slow-model execution") + return remaining + + def _retry_wait(self, seconds): + if seconds >= self._remaining(): + raise TaskBudgetExceeded("Task budget cannot accommodate another slow-model retry") + time.sleep(seconds) def json_request(self, messages, validate, *, role): credentials = self.credentials @@ -84,18 +168,35 @@ def json_request(self, messages, validate, *, role): "model": credentials.get("model") or "unknown", "messages": messages, "stream": False, - "max_tokens": self.config.slow_max_tokens, + "max_tokens": (min(self.config.slow_max_tokens, self.config.slow_verify_max_tokens) + if role == "verify_stage" else self.config.slow_max_tokens), } + if role == "gui_choice": + payload["max_tokens"] = min(self.config.slow_max_tokens, 2048) effort = self.config.slow_reasoning_effort + if role == "gui_choice": + effort = "low" + if role == "verify_stage" and self.config.slow_verify_reasoning_effort is not None: + effort = self.config.slow_verify_reasoning_effort if effort == "off": payload["enable_thinking"] = False elif effort is not None: payload["reasoning_effort"] = effort + image_stats = (_pack_images(payload, self.config.slow_image_quality, + self.config.slow_max_request_bytes) + if self.config.execution_mode == "goal" else {}) def build_request(): + data = json.dumps(payload).encode("utf-8") + # Repairs can append messages after image packing; enforce the limit on every attempt. + if self.config.execution_mode == "goal" and len(data) > self.config.slow_max_request_bytes: + raise ValueError( + "slow model request body exceeds slow_max_request_bytes: " + f"{len(data)} > {self.config.slow_max_request_bytes} bytes" + ) return urllib.request.Request( credentials["base_url"].rstrip("/") + "/chat/completions", - data=json.dumps(payload).encode("utf-8"), + data=data, headers={ **( {"Authorization": "Bearer " + key["api_key"]} @@ -125,6 +226,7 @@ def audit(event, started, attempt, **details): "time": time.time(), "elapsed_seconds": round(time.perf_counter() - started, 3), "request_bytes": len(request.data), + **image_stats, **details, } text = json.dumps(self.key_pool.redact(record), ensure_ascii=False) @@ -152,13 +254,14 @@ def public_headers(headers): attempt = 0 failures = 0 while True: + timeout = min(self.config.slow_timeout_seconds, self._remaining()) attempt += 1 started = time.perf_counter() request = build_request() # 每次重试重建:升档思考强度需重新序列化 audit("request_start", started, attempt) try: try: - with opener(request, timeout=self.config.slow_timeout_seconds) as response: + with opener(request, timeout=timeout) as response: status = response.status headers = public_headers(response.headers) raw = json.load(response) @@ -209,7 +312,7 @@ def public_headers(headers): raise error delay = retry_after_seconds(headers.get("retry-after"), 30) audit("retry_wait", started, attempt, retry_wait_seconds=delay) - time.sleep(delay) + self._retry_wait(delay) continue choices = raw.get("choices") or [] if not choices: @@ -221,20 +324,23 @@ def public_headers(headers): finish_reason=finish_reason, usage=raw.get("usage")) if finish_reason == "length": completion_tokens = (raw.get("usage") or {}).get("completion_tokens") - raise ValueError( + raise ModelResponseError( f"slow model response truncated by max_tokens (finish_reason=length, " f"completion_tokens={completion_tokens}); raise slow_max_tokens" ) if not content and choice.get("message", {}).get("reasoning_content"): - raise ValueError( + raise ModelResponseError( "slow model returned empty content (all completion tokens stayed in the " "reasoning channel); escalating reasoning_effort for the next attempt" ) value = extract_json_object(content) if value is None: - raise ValueError("slow model response contains no JSON object") + raise ModelResponseError("slow model response contains no JSON object") if not validate(value): - raise ValueError("slow model response is missing required fields") + correction = messages[-1].get("content", "") if messages else "" + detail = (correction if isinstance(correction, str) + and correction.startswith("Invalid") else "") + raise ModelResponseError("slow model response failed validation: " + detail[:600]) return value except (urllib.error.URLError, ConnectionError, TimeoutError, ValueError) as error: audit( @@ -249,7 +355,15 @@ def public_headers(headers): raise # 内容级失败(空正文/截断/非法 JSON)时升档思考强度后重试,不原样重发 current_effort = payload.get("reasoning_effort") - if current_effort in _REASON_EFFORT_LADDER: + if (self.config.execution_mode == "goal" and role == "plan_stage" + and "finish_reason=length" in str(error)): + payload["reasoning_effort"] = "low" + messages.append({"role": "user", "content": + "The previous response was truncated and NOTHING from it was executed. " + "Return one COMPLETE compact goal-stage JSON object, without reasoning or " + "a long script. Use one bounded edit/read, reuse memory, and preserve the " + "original requirements. Do not repeat the unbounded response."}) + elif current_effort in _REASON_EFFORT_LADDER: payload["reasoning_effort"] = _REASON_EFFORT_LADDER[current_effort] audit("retry_wait", started, attempt, retry_wait_seconds=30) - time.sleep(30) + self._retry_wait(30) diff --git a/src/tandem/planning/goal_planner.py b/src/tandem/planning/goal_planner.py new file mode 100644 index 0000000..473f37f --- /dev/null +++ b/src/tandem/planning/goal_planner.py @@ -0,0 +1,370 @@ +"""One self-contained subgoal, not a speculative list of future UI actions.""" + +import base64 +import json +import re + +from ..memory.context import _fact_view, planning_context + +REQUIREMENTS = """Extract the ORIGINAL task into JSON {requirements:[{id,requirement,output_path, +kind,units,unit_fact_keys,facts_needed}]}. kind is delivery or constraint. units is a list of STRINGS with +delivery units with their completion criteria; enumerate explicitly requested pages/items, otherwise +use one unit. Constraints have units:[]. facts_needed is [{key,description}] naming only essential +stable source information acquired from supplied materials; reuse the SAME key across requirements. +Never register current focus, open windows or selection as required source facts. Split large source +packets into the concrete fields or small sections needed by individual deliverables. unit_fact_keys +is an array parallel to units: each entry lists ONLY the facts_needed keys required by THAT unit. +Keys may reference definitions in ANY requirement; define each shared source key once. Use [] for a +unit needing no source reading. Constraints MUST have units:[] AND unit_fact_keys:[], not [[]]. +Include a constraint's required source key on the delivery units that must satisfy that constraint. +A source-reading dependency must not gate unrelated work. +Do not list facts already stated in the instruction as facts_needed. Names, titles, URLs and target +systems explicitly provided by the user are inputs, not missing source facts. +Keep all original quantities, locations, prohibitions, interaction requirements and saving conditions. +Enumerate the individual pages/slides/form sections as small delivery units, including when their count +is explicit but their names are not. Do not collapse 'all 26 slides' into one giant unit. Each unit +lists only its own missing source content; whole-document consistency remains a final constraint. +output_path is null unless literally given as an absolute output path in the instruction. +Do not prescribe actions, guess source facts, inspect evaluators, or introduce hidden grading criteria. +Return a compact contract, no reasoning, no speculative extra work.""" + +REQUIREMENTS += ''' +Every requirement MUST contain ALL seven keys, including a plain-string "requirement" describing the +actual obligation (also for constraints). units MUST be an array of STRINGS, never objects. +Requirement IDs must be nonempty and UNIQUE across the complete contract, including constraints. +Exact shape example: {"requirements":[{"id":"R1","requirement":"Create and save the requested document", +"output_path":null,"kind":"delivery","units":["Requested document is complete and saved"], +"unit_fact_keys":[[]], +"facts_needed":[]}]}. This example illustrates the format only; extract the real task's obligations. +''' + +PLAN = """Delegate ONE meaningful bounded subtask using the host's task memory. You are the planner; +the fast GUI choice model will autonomously observe and act, not follow an action list. Return JSON: +{done:false,work_item_id:"exact ready id",route:"gui|cli",purpose:"prepare|produce|save|verify", +goal:"specific result",completion_criteria:"stage exit condition",method:"concise approach", +input_fact_keys:["known key"],inputs:[{label:"field meaning",target_hint:"observed field label", +text:"exact supplied text", +input_mode:"replace|insert"}],guidance:"brief necessary navigation/context, not an ordered click script", +max_actions:24,command:"CLI only; otherwise empty string", +recovery_strategy:"continue|inspect_state|restore_context|alternate_interaction|alternate_route"}. +When ready_work_items[].recovery_required is true, choose an explicit recovery_strategy other than +continue. Changing the wording of a goal/method does not change the execution path. The host retains +failed navigation families for the observed execution scope across stages; unrelated source facts do +not release them. Restore the execution context, inspect a missing capability, use another interaction +or switch route only when the original task permits it. Never repeat an already-executed action merely +because a receipt or observation is missing. +Use a ready work item's FIXED goal and original requirements. Do not rename units or lower final criteria. +Missing source facts are acquisition targets, NOT a hard gate on starting a delivery unit. First make +and save the useful portion supported by known facts; leave unknown values unresolved, never guess them. +Choose a bounded incremental stage exit condition, not the whole unit's acceptance criteria. A partial +stage may succeed while the unit remains incomplete. Read a missing source only before the next action +actually needs it, then resume the preserved artifact. Respect explicit user-required reading order. +input_fact_keys may name ONLY confirmed task-scope keys present in memory.known_facts, NOT keys that +are merely requested in facts_needed. Use [] before any source facts have been observed. +For prepare, acquire only missing essential facts; do not reread facts already in memory. Source notes +are historical evidence, not guaranteed current UI. Reuse confirmed task facts. Temporary inventory, +selection and page state must be reobserved. Do not fabricate a URL or missing personal information. +Bound visual source reading to one small section at a time (normally 4-6 observations); the stage-end +verifier will record its contents. Do not delegate reading several documents and navigating a new site +as a single preparation goal. Later preparation continues from materials.covered_parts, not page one. +For GUI give a whole local goal (a form section, adding an attachment, a settings operation), normally +8-24 possible actions, not merely 'open menu'. Bound editing to the CURRENT page/form section or slide. +Stop before a new page requires new values or reasoning; plan it from its actual observation next. +Give only ONE current value per target_hint. Reading multiple URLs in the same address bar requires +separate stages, not competing replacement inputs. A typed navigation URL still requires Enter. +A file chooser may be completed with an already supplied exact path. Never delegate uploading while +the current form still needs unknown required fields. First fill the known current fields, then stop +to acquire only the missing values. Provide ALL exact text values from the instruction or known facts. +For every text input, target_hint must identify its intended observed field (copy the visible label +when available). For browser navigation use the observed address-bar label; for a document body or +desktop launcher identify that specific editor/search field. Text is never interchangeable among fields. +The choice model cannot generate free text. It can click, type provided inputs, use keyboard, scroll, +wait, request stage verification, or return control. Missing data/new reasoning requires another stage. +For CLI provide one bounded Bash command, current inputs and expected real results. Prefer actually +producing/saving deliverables over repeatedly reading tutorials, building audits or protecting unchanged +files. Inspect edited documents in the actual application before claiming them verified. Keep source +and final paths distinct. CLI is forbidden if allow_cli=false. Task constraints override routing choices. +Use installed tools; do not assume OCR binaries or passwordless sudo. Do not install packages as a +substitute for source reading. Keep commands valid Bash, including background-command separators. +memory.observed_missing_modules records actual import failures: use an available alternative, not the +same missing package again. Derive edit preconditions from observed document structure, not guessed +replacement counts. After an assertion fails, narrow the next edit using the real error and partial output. +Do not inspect evaluator code, ground truth, rewards, hidden state, unrelated secrets or scoring APIs. +Previous methods/errors are in memory; changing their description does not make them new. If stalled, +choose another method or another independent ready unit. Avoid optional research and cosmetic detours. +Only return {done:true,task_status:"completed|infeasible",current_goal:"evidence-based reason"} for an +actual whole-task terminal claim. Stage completion is NOT task completion. A blocked unit is not proof +the task is infeasible. Keep the response short: goal, inputs and one bounded command, no long analysis.""" + +PLAN += """ +Treat capability_missing and unresolved execution scopes as prerequisites, not wording problems. +memory.pending_publications are edited working copies not yet delivered. Publish a useful saved increment +to the observed user deliverable and reopen it before starting more optional research or another edit. +Saving /tmp files is not delivery. After a bounded edit, prefer publishing that increment rather than +spending the remaining budget on tiny edits in an unpublished copy. Do not overwrite it from a stale GUI. +Keep a valuable unsaved form in its own tab when opening a reference; explicitly preserve that context. +Use remaining_wall_seconds as a real deadline. Publish a useful current increment early; near the deadline, +prefer final-path saving/reopening over optional source exploration or another tiny unpublished edit. +Partial work is not task completion: retain unmet requirements and let official evaluation judge it. +Opening a file means it is loaded and usable in the requested application. Complete necessary native +open/import confirmations; handing off a path or leaving an import preview is not the finished handoff. +memory.active_blocker is the current observed execution obstruction, shared across work units. +Changing its name, unit or recovery_strategy does not resolve it. GUI recovery must expose an actually +available target/interaction; after grounding is exhausted use existing observed controls or another +allowed route. No new speculative coordinates or repeated requests for the exhausted capability. +Give goal as an action-oriented bounded objective, and completion_criteria as a SEPARATE desired +observable outcome. An exit condition is not an observed fact. Do not invent folders or navigation +capabilities: a page number printed in source content does not establish a viewer pagination control. +Resolve the active obstruction in ONE bounded prepare stage before proposing edits behind it. A business +dialog needed for the task must be completed, not dismissed. Do not reissue an unbound input unchanged: +first navigate to expose its target, acquire a supported interaction, or use another allowed route. +After CLI edits, the already-open GUI may be stale. Close/reopen the saved working artifact without +overwriting it from the old GUI buffer before visual acceptance. A title alone does not establish freshness. +Prefer a ready delivery unit over more preparation. Preparation is bounded acquisition for ONE missing +field or source section, not an all-document audit or a pixel-analysis research project. State the exact +observable subgoal, never 'Acquire the required source facts'. Reuse saved extractions and artifacts. +After two stages without new required facts or delivery progress, use a genuinely different method or +another ready unit. Do not add optional diagnostic outputs as new completion requirements. Make and +save a usable increment as soon as its inputs are known; preserve all original final requirements. +historical_ui_notes are prior observations, NOT the current interface. Current screenshot/controls win. +For prepare, follow next_acquisition: acquire the stated missing registered fields for that ONE unit. +Do not inventory every dropdown or inspect source code before filling fields whose values are known. +Use exact registered keys for acquired contents; fact_index lists already observed auxiliary sources, +not new dependencies. Do not reread auxiliary sources solely because their full text is omitted here. +Never type into an unrelated/Untitled document. Confirm the target application and document first. +For editing an existing document, include target_document:"exact filename" on GUI produce/save stages. +If it is not currently open in the foreground, first use a prepare stage to open/switch to it. A +navigation-only prepare stage may have no source fact targets. For websites or creating a NEW document, +target_document may be null. Available GUI operations include right-click, double-click, Escape, +Alt+Tab, Ctrl+O, keyboard navigation, supplied text, scrolling and screenshot-grounded targets. +known_facts includes partial and auxiliary source content; complete:false is usable for a partial +increment but does NOT mean the source or unit is complete. Never guess ownership/identity when joining +records from different sources. Do not keep preparing solely to make every facts_needed entry complete. +If an observed fact is indexed but its text is omitted/excerpted, return ONLY {read_memory:["key",...]} +for up to six keys, at most twice before planning. This reads stored evidence without reopening files. +Failure constraints, remaining work and rejected memory updates are actionable state, not suggestions +to repeat the same attempt with different wording. When the executor reports a blocked cycle, choose +another interaction, another source or an independent ready unit. +For a GUI stage that needs to open or switch applications, include target_application:"exact app name". +This explicit input will also be available for typing in the desktop launcher; do not leave it only in +prose guidance. Set it to null when not applicable. Prefer CLI to open an installed application or a +known document when allowed, then delegate actual UI editing to JEV. Do not alternate the application +drawer open/closed or repeat a covered target. Resolve the visible overlay or use another interaction. +completed milestones are delivered sections, not instructions to recreate them. Continue with an unmet +section/field group even when the whole unit is incomplete; do not re-audit completed increments. +For native input_type=date, provide ISO YYYY-MM-DD from the task/source. The executor edits the observed +Day/Month/Year segments separately; never concatenate a guessed locale's month/day/year digits. +When a date is still empty after typing, use its calendar controls or revise the input; don't retype it. +Two consecutive CLI execution failures require one bounded read-only structure inspection or GUI work +before another write. Use actual object identities; do not weaken assertions to obtain a pass. +memory.artifacts is the cumulative working-copy registry: resume its existing working_path for the +same target_path, not a fresh copy of the original on each stage. Save increments to that same artifact. +Do not replace editable slides, animations or native objects with full-page reference screenshots. +Treat website application code as read-only unless the user explicitly asks to develop that application. +Do not edit HTML/JS source, inject bootstraps, add options, remove fields or bypass validation to fill +a website. Use its real UI; ordinary CLI editing of the requested output document remains allowed. +For a supplied command entered into an application's command/input bar, add kind:"command",commit:"enter" +to that input. Ordinary text uses kind:"text",commit:null; a known selection uses kind:"choice". +These describe one observed control interaction, not a sequence of speculative future clicks. +unsupported_methods records ACTUAL application errors, not suggestions. Do not keep changing arguments +of an unrecognized command; use another supported interaction in the same required application. +Memory failures.error is observed evidence; suggested_actions is only a hypothesis. Check it against +the real application before treating it as valid syntax. Preserve already confirmed source content. +Window opening/maximization/menu navigation are prepare, not produce or saved-delivery milestones. +Keep the stage exit condition separate from the fixed unit criterion and whole-task requirements. +""" + + +def goal_requirements(client, instruction): + messages = [{"role": "system", "content": REQUIREMENTS}, {"role": "user", "content": instruction}] + + def valid(v): + okay = (isinstance(v.get("requirements"), list) and bool(v["requirements"]) + and all(isinstance(r, dict) and isinstance(r.get("id"), str) + and isinstance(r.get("requirement"), str) + and r.get("kind") in {"delivery", "constraint"} + and isinstance(r.get("units"), list) + and (r.get("kind") == "constraint" or bool(r["units"])) + and all(isinstance(u, str) for u in r["units"]) + and isinstance(r.get("unit_fact_keys"), list) + and len(r["unit_fact_keys"]) == len(r["units"]) + and all(isinstance(keys, list) and all(isinstance(k, str) for k in keys) + for keys in r["unit_fact_keys"]) + and isinstance(r.get("facts_needed"), list) + and all(isinstance(f, dict) and isinstance(f.get("key"), str) + and isinstance(f.get("description"), str) for f in r["facts_needed"]) + for r in v["requirements"])) + if okay: + ids = [r["id"].strip() for r in v["requirements"]] + if not all(ids) or len(set(ids)) != len(ids): + messages.append({"role": "user", "content": + "Duplicate or empty requirement id. Give EVERY requirement a unique nonempty id; " + "preserve ALL delivery obligations and return the COMPLETE corrected contract."}) + return False + registered = {f["key"] for r in v["requirements"] for f in r["facts_needed"]} + okay = all(set(keys) <= registered for r in v["requirements"] for keys in r["unit_fact_keys"]) + if not okay: + messages.append({"role": "user", "content": + "Invalid requirements JSON. Every entry needs id, requirement (plain text, mandatory), " + "output_path, kind, units (array of STRINGS, not objects), unit_fact_keys " + "(one array of source key strings per unit) and facts_needed " + "(array of {key,description}). Include constraint text in requirement; units:[] is not " + "a substitute. Constraints need units:[] AND unit_fact_keys:[], not [[]]. " + "Shared source keys may be defined in any requirement. Return the COMPLETE object " + "{requirements:[...]}, not an isolated fact or requirement."}) + return okay + + value = client.json_request(messages, valid, role="task_requirements") + for r in value["requirements"]: + path = r.get("output_path") + if not isinstance(path, str) or not path.startswith("/") or path not in instruction: + r["output_path"] = None + return value["requirements"] + + +def goal_error(value, memory, allow_cli): + """Name the rejected field/constraint so JSON repair can actually change the request.""" + if type(value.get("done")) is not bool: + return "done must be a boolean" + if value["done"]: + return ("" if value.get("task_status") in ("completed", "infeasible") + and isinstance(value.get("current_goal"), str) and value["current_goal"].strip() + else "terminal claims require task_status and a nonempty current_goal") + ready = {i["id"]: i for i in memory.ready_items()} + if not isinstance(value.get("work_item_id"), str): + return "work_item_id must be a string naming an exact ready unit" + item = ready.get(value.get("work_item_id")) + if item is None: + return "work_item_id must be one of " + json.dumps(list(ready)) + strategy = value.get("recovery_strategy", "continue") + if strategy not in {"continue", "inspect_state", "restore_context", "alternate_interaction", + "alternate_route"}: + return "recovery_strategy must name a supported execution recovery" + if item.get("recovery_required") and strategy == "continue": + return "execution recovery is required; declare how the blocked path will change" + if item.get("stalled_stages", 0) >= 2: + previous = item.get("last_attempt", {}) + if (value.get("route") == previous.get("route") == "cli" + and isinstance(value.get("command"), str) + and value["command"].strip() == previous.get("command", "").strip()): + return "command repeats the same CLI attempt after two stalled stages; change method or work unit" + if value.get("route") not in (("gui", "cli") if allow_cli else ("gui",)): + return "route must be gui" + (" or cli" if allow_cli else "; CLI is disabled") + purposes = ("prepare",) if item["kind"] == "prepare" else ("prepare", "produce", "save", "verify") + if value.get("purpose") not in purposes: + return "purpose must be one of " + json.dumps(sorted(purposes)) + if (item.get("failed_cli_stages", 0) >= 2 and value.get("route") == "cli" + and value.get("purpose") in {"produce", "save"}): + return ("two CLI executions failed; inspect structure read-only in prepare " + "or use GUI before writing") + for key in ("goal", "completion_criteria", "method"): + if not isinstance(value.get(key), str) or not value[key].strip(): + return key + " must be a nonempty string" + if not isinstance(value.get("guidance"), str): + return "guidance must be a string (empty allowed)" + for key in ("target_document", "target_application"): + if value.get(key) is not None and (not isinstance(value[key], str) or not value[key].strip()): + return key + " must be a nonempty string or null" + app_words = re.findall(r"\w{3,}", (value.get("target_application") or "").casefold()) + blockers = [c for c in memory.execution_constraints if c.get("kind") == "execution_checkpoint" + and c.get("global_scope") and c.get("recovery_required") + and (not app_words or any(w in json.loads(c["scope_key"])[0].casefold() for w in app_words))] + if blockers and value.get("route") == "gui" and value.get("purpose") in {"produce", "save"}: + return "resolve the active GUI input obstruction in a bounded prepare stage before editing" + if type(value.get("max_actions")) is not int or not 1 <= value["max_actions"] <= 24: + return "max_actions must be an integer in 1..24" + allowed = [k for k, f in memory.facts.items() if f["status"] == "confirmed" and f["scope"] == "task"] + if not isinstance(value.get("input_fact_keys"), list) or any( + not isinstance(k, str) or k not in allowed for k in value["input_fact_keys"]): + return ("input_fact_keys must contain only acquired confirmed task keys: " + json.dumps(allowed) + + "; copy keys EXACTLY, including dots and punctuation; do not normalize filenames. " + "Use [] if none are needed. Rejected keys: " + json.dumps(value.get("input_fact_keys"))) + if not isinstance(value.get("inputs"), list): + return "inputs must be an array (empty allowed)" + input_values = {} + for index, entry in enumerate(value["inputs"]): + if (not isinstance(entry, dict) or not isinstance(entry.get("label"), str) + or not isinstance(entry.get("text"), str) + or entry.get("input_mode") not in ("replace", "insert")): + return f"inputs[{index}] requires label:string, text:string, input_mode:replace|insert" + if "target_hint" in entry and (not isinstance(entry["target_hint"], str) + or not entry["target_hint"].strip()): + return f"inputs[{index}].target_hint must identify the intended field" + if entry.get("kind", "text") not in {"text", "command", "choice"}: + return f"inputs[{index}].kind must be text, command or choice" + if entry.get("commit") not in (None, "enter"): + return f"inputs[{index}].commit must be enter or null" + blocked = memory.blocked_command(entry["text"], value.get("target_application")) + if blocked and value["route"] == "gui": + return blocked + hint = entry.get("target_hint", entry["label"]).strip().casefold() + if value["route"] == "gui" and hint in input_values and input_values[hint] != entry["text"]: + return f"inputs assign conflicting texts to {hint!r}; delegate one current value/URL per stage" + input_values[hint] = entry["text"] + previous = item.get("last_attempt", {}) + if (item.get("stalled_stages", 0) >= 2 + and previous.get("failure_kind", "").startswith("capability_missing:") + and value["route"] == "gui" and value["inputs"] + and [i["text"] for i in value["inputs"] if i.get("scope") != "desktop_launcher"] + == [i["text"] for i in previous.get("inputs", []) if i.get("scope") != "desktop_launcher"]): + return "the same input is unsupported; expose its target in a prepare stage or change route" + if not isinstance(value.get("command"), str): + return "command must be a string; use an empty string for GUI" + if value["route"] == "cli" and not value["command"].strip(): + return "command must be a nonempty Bash command for CLI" + return "" + + +def valid_goal(value, memory, allow_cli): + return not goal_error(value, memory, allow_cli) + + +def plan_goal(client, instruction, screenshot, targets, recent, tree, verified_focus=None, + requirements=None, remaining_steps=None, *, memory, allow_cli=True, route_policy="auto", + remaining_seconds=None): + allow_cli = allow_cli and not re.search( + r"do not (?:open or use |use |open )?(?:terminal|the terminal)|\bGUI[- ]only\b|\bCLI is forbidden\b", + instruction, re.I) + context = {"task": instruction, "memory": planning_context(memory), "allow_cli": bool(allow_cli), + "route_policy": route_policy, "remaining_actions": remaining_steps, + "remaining_wall_seconds": remaining_seconds, + "visible_controls": [t.compact() for t in targets], "recent": recent[-4:]} + messages = [{"role": "system", "content": PLAN}, {"role": "user", "content": [ + {"type": "text", "text": json.dumps(context, ensure_ascii=False)}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + + base64.b64encode(screenshot.read_bytes()).decode("ascii")}}, + ]}] + reads = 0 + + def validate(v): + if set(v) == {"read_memory"}: + okay = (reads < 2 and isinstance(v["read_memory"], list) and 1 <= len(v["read_memory"]) <= 6 + and all(isinstance(k, str) and k in memory.facts and memory.facts[k]["scope"] == "task" + for k in v["read_memory"])) + error = ("read_memory needs 1..6 existing task-scope keys and at most two reads" + if not okay else "") + else: + error = goal_error(v, memory, allow_cli) + if error: + messages.append({"role": "user", "content": + "Invalid goal-stage JSON: " + error + ". Return the COMPLETE corrected compact stage " + "object, preserving evidence and original requirements. Do not replay an executed action."}) + return not error + + while True: + value = client.json_request(messages, validate, role="plan_stage") + if set(value) != {"read_memory"}: + application = value.get("target_application") + if not value["done"] and value["route"] == "gui" and application and not any( + i["text"] == application for i in value["inputs"]): + value["inputs"].append({"label": "Target application name for desktop search", + "target_hint": "Desktop search", "text": application, + "input_mode": "replace", "scope": "desktop_launcher"}) + return value + reads += 1 + messages.append({"role": "assistant", "content": json.dumps(value)}) + messages.append({"role": "user", "content": json.dumps({ + "stored_source_facts": [_fact_view(memory.facts[k], 16000) for k in value["read_memory"]], + "instruction": "Use the stored evidence; now return a bounded stage plan. " + "Do not reopen these sources just to recover the same content.", + "memory_reads_remaining": 2 - reads}, ensure_ascii=False)}) diff --git a/src/tandem/runtime/episode.py b/src/tandem/runtime/episode.py index bb4dbeb..73efc3c 100644 --- a/src/tandem/runtime/episode.py +++ b/src/tandem/runtime/episode.py @@ -12,6 +12,7 @@ from pathlib import Path from ..gui.focus import action_key, toggle_transition +from ..memory.context import verification_context from ..protocol.models import CliAction, FailAction, FinishAction, Observation from .stopping import NO_ACTION_LIMIT @@ -75,6 +76,7 @@ def terminal_decision(session, plan, observation: Observation, tree, output_file control_state_check=unchanged, requirements=session.requirements, output_files=output_files, + **({"task_memory": verification_context(session.goal.memory)} if session.goal is not None else {}), ) session.events.append( { diff --git a/src/tandem/runtime/goal_stage.py b/src/tandem/runtime/goal_stage.py new file mode 100644 index 0000000..6af3ad5 --- /dev/null +++ b/src/tandem/runtime/goal_stage.py @@ -0,0 +1,494 @@ +"""Goal-stage loop: receipts/observations each step, one semantic verifier at stage end.""" + +import json +import re +import time +import xml.etree.ElementTree as ET +from pathlib import Path + +from ..gui.goal_candidates import ( + application_mismatch, + foreground_tree, + goal_candidates, + launcher_input, + portal_needs_grounding, +) +from ..gui.goal_candidates import ( + document_mismatch as _document_mismatch, +) +from ..gui.interaction import target_effect as _target_effect +from ..gui.interaction import visual_changed as _visual_changed +from ..gui.interaction import visual_identity +from ..memory.context import execution_context +from ..models.ranking import ranking_dict +from ..models.slow import ModelResponseError +from ..protocol.models import CliAction, FailAction +from .episode import handle_done, resolve_done_probe + + +class ObservationRequired(Exception): + """Refresh without executing/replaying an action or consuming an action budget.""" + + +def _state_key(targets): + return tuple(sorted((t.role, t.name, t.x, t.y, t.value, bool(t.focused), str(t.checked), str(t.selected)) + for t in _progress_targets(targets))) + + +def _progress_targets(targets): + # Shell display text (clock/battery) is not task progress. Keep shell controls and focused input. + return [t for t in targets if not ( + (t.application == "gnome-shell" or "gnome-shell" in t.parent_context) + and t.role in {"text", "label", "static"} and not t.focused)] + + +def _loop_state(targets, tree): + root = ET.fromstring(tree or "") + return (root.get("foreground_title", ""), root.get("foreground_class", ""), + tuple(sorted((t.role, t.semantic_key if "dom" in t.source else _control_name(t), + 0 if "dom" in t.source else t.x, 0 if "dom" in t.source else t.y, + "" if t.role == "slider" else t.value, + str(t.checked)) for t in _progress_targets(targets) + if t.application != "gnome-shell" and "gnome-shell" not in t.parent_context)), + portal_needs_grounding(tree)) + + +def _control_name(target): + if target.role != "page_tab": + return target.name + name = re.sub(r"\s+- Memory usage - .+$", "", target.name) + return re.sub(r"\s*[–-]\s*(?:Audio playing|Video playing in picture-in-picture mode)$", "", name) + + +def _cycle_actions(recent, state): + for period in (2, 3, 4): + if len(recent) < period * 2: + continue + first, second = recent[-period * 2:-period], recent[-period:] + if first == second and any(a != first[0] for a in first): + return {a["key"] for a in second if a["state"] == state} + return set() + + +def _action_key(candidate, targets=()): + target = next((t for t in targets if t.id == candidate.target_id), None) + name = ((target.semantic_key if "dom" in target.source else + f"{target.role}|{_control_name(target)}|{target.x}|{target.y}") if target else + visual_identity(candidate) or candidate.metadata.get("semantic_key") or candidate.description) + if candidate.operation == "keypress": + name += repr([(t.role, t.name, t.x, t.y) for t in targets if t.focused]) + parameters = {k: candidate.metadata[k] + for k in ("text", "input_mode", "fraction", "key_sequence", "visual_point") + if k in candidate.metadata and not (k == "visual_point" and visual_identity(candidate))} + return (candidate.operation, name, json.dumps(parameters, sort_keys=True, ensure_ascii=False)) + + +def _navigation_key(candidate, target, work, state): + if candidate.operation == "scroll" or candidate.id in {"KEYPRESS_PAGE_UP", "KEYPRESS_PAGE_DOWN"}: + return (work, state[1], "browse", "up" if "UP" in candidate.id.upper() else "down") + if candidate.source == "a11y_menu": + return (work, state[1], "menu_navigation", candidate.description) + if target and target.scope == "browser_chrome" and candidate.operation in {"click", "double_click"}: + return (work, state[1], "browser_navigation", target.name.casefold()) + if target and (target.application == "gnome-shell" or "gnome-shell" in target.parent_context): + return (work, state[1], "shell_navigation", "system_controls") + if target and re.search(r"\b(?:zoom|pan)\b", target.name, re.I): + return (work, state[1], candidate.operation, target.name.casefold()) + return None + + +class GoalExecution: + def __init__(self, memory, max_actions, interaction): + self.memory = memory + self.max_actions = max_actions + self.interaction = interaction + self.pending = None + self.history = [] + self.end_reason = None + self.visual = [] + self.visual_frame = None + self.recovery_stage = None + self.recovery_calls = 0 + self.recovery_seconds = 0.0 + self.attempts = [] + self.failed_actions = {} + self.grounded_states = {} + self.no_effect = 0 + self.empty_stages = 0 + + def _emit_state(self, session, step): + session.events.append({"route": "task_memory", "step": step, + "state": self.memory.snapshot()}) + + def _receive(self, session, observation, targets, tree): + if self.pending is None: + return + receipt = observation.last_receipt + if receipt is None or not receipt.acknowledged: + raise RuntimeError("Missing execution receipt; action is not replayed") + session.consecutive_no_action = 0 + old = self.pending + self.pending = None + signature = _state_key(targets) + changed = old["kind"] == "cli" or signature != old["state"] or _visual_changed( + old["before_screenshot"], observation.screenshot.path) + if old.get("effect_bounds") and old["operation"] == "type_text": + changed = changed or _visual_changed(old["before_screenshot"], observation.screenshot.path, + old["effect_bounds"]) + effect = _target_effect(old, targets) + if effect == "met": + self.recovery_calls, self.recovery_seconds = 0, 0.0 + relevant_change = (old["kind"] == "cli" or effect == "met" + or effect != "not_met" and old["loop_state"] != _loop_state(targets, tree)) + if effect == "unknown" and (not targets or old.get("visual_action")): + relevant_change = changed + if old.get("overlay_bounds") and effect != "met": + relevant_change = _visual_changed(old["before_screenshot"], observation.screenshot.path, + old["overlay_bounds"]) + self.interaction.receive(old, effect, changed, receipt.returncode, targets=targets, + evidence_ref=f"A{observation.step_idx}") + self.memory.active_stage["interaction"] = self.interaction.snapshot() + if old["kind"] != "cli": + key = (self.memory.active_stage["work_item_id"], old["loop_state"], old["candidate_key"]) + if not relevant_change: + self.failed_actions[key] = self.failed_actions.get(key, 0) + 1 + self.attempts.append({"work": key[0], "state": old["loop_state"], "key": old["candidate_key"]}) + self.attempts = self.attempts[-16:] + self.no_effect = 0 if relevant_change else self.no_effect + 1 + record = {"step": observation.step_idx, "evidence_ref": f"A{observation.step_idx}", + "kind": old["kind"], "operation": old["operation"], "target": old["target"], + "action": old["action"], "returncode": receipt.returncode, + "stdout": receipt.stdout, "stderr": receipt.stderr, + "full_output_ref": receipt.full_output_ref, "screenshot": observation.screenshot.path, + "observed_controls": [t.compact() for t in targets][:40], + "source_frame": self.memory.active_stage["purpose"] == "prepare", + "signature": signature, "observed_change": changed, + "relevant_change": relevant_change, "target_effect": effect, + "interaction_changed": changed, "local_effect": self.interaction.local_effect, + "interaction": self.interaction.snapshot(), + "application": ET.fromstring(tree or "").get("foreground_class", ""), + "navigation_key": old.get("navigation_key")} + self.history.append(record) + self.memory.record_action(record) + session.recent.append({"operation": old["operation"], "target": old["target"], + "goal": self.memory.active_stage["goal"], + "outcome": "changed" if relevant_change else "no_effect", + "returncode": receipt.returncode, + "evidence_ref": record["evidence_ref"]}) + session.events.append({"route": "goal_action", "step": observation.step_idx, + "record": record, "effect_verified": False}) + if receipt.returncode != 0: + self.end_reason = "execution_error" + elif old["kind"] == "cli": + self.end_reason = "cli_returned" + elif self.memory.active_stage["actions"] >= self.memory.active_stage["max_actions"]: + self.end_reason = "stage_action_limit" + elif self.interaction.checkpoint_due: + self.end_reason = "execution_checkpoint_stalled" + elif self.no_effect >= 3 and (session.components.grounder is None + or (self.memory.active_stage["work_item_id"], _loop_state(targets, tree)) + in self.grounded_states): + self.end_reason = "repeated_action_without_observable_progress" + + def _finish(self, session, observation): + reference = f"{self.memory.active_stage['id']}:O{observation.step_idx}" + evidence = {h["evidence_ref"]: h for h in self.history} + evidence[reference] = {"screenshot": observation.screenshot.path} + started = time.perf_counter() + result = session.components.stage_verifier( + session.instruction, Path(observation.screenshot.path), memory=self.memory, + history=[{k: v for k, v in h.items() if k != "signature"} for h in self.history], + end_reason=self.end_reason, evidence=evidence) + stage_id = self.memory.active_stage["id"] + work = self.memory.active_stage["work_item_id"] + source_progress = (self.memory.stats["new_facts"], self.memory.stats["source_parts"]) + delivery = (self.memory.stats["delivery_advances"], self.memory.stats["milestone_advances"]) + self.interaction.finish(self.memory, result, self.history) + advanced = self.memory.apply(result, evidence) + acquired = source_progress != (self.memory.stats["new_facts"], self.memory.stats["source_parts"]) + delivered = delivery != (self.memory.stats["delivery_advances"], + self.memory.stats["milestone_advances"]) + if delivered: + self.recovery_calls, self.recovery_seconds = 0, 0.0 + self.empty_stages = self.empty_stages + 1 if not self.history and not advanced and not acquired else 0 + if self.memory.work_items[work]["status"] == "done": + self.failed_actions = {k: v for k, v in self.failed_actions.items() if k[0] != work} + self.attempts = [a for a in self.attempts if a["work"] != work] + self.memory.execution_constraints = [c for c in self.memory.execution_constraints + if c["work_item_id"] != work or c.get("global_scope")] + for rejected in self.memory.last_rejections: + session.events.append({"route": "memory_update_rejected", "stage_id": stage_id, **rejected}) + session.events.append({"route": "stage_verified", "step": observation.step_idx, + "stage_id": stage_id, "end_reason": self.end_reason, + "evidence": {k: {"screenshot": v["screenshot"], + "full_output_ref": v.get("full_output_ref")} + for k, v in evidence.items()}, + "verification": result, "memory_advanced": advanced, + "progress_delta": {"source": acquired, "delivery": delivered}, + "control_decision": "completed_unit" if self.memory.work_items[work]["status"] + == "done" else "recover_execution" if self.memory.work_items[work].get( + "recovery_required") else "continue_increment" if advanced else "replan", + "seconds": time.perf_counter() - started}) + session.recent.append({"operation": "stage_verified", "stage_id": stage_id, + "outcome": result["stage_outcome"], "evidence": result["summary"], + "remaining": result["remaining"]}) + self.history, self.end_reason, self.visual = [], None, [] + self.visual_frame = None + self.no_effect = 0 + self._emit_state(session, observation.step_idx) + if result.get("failure_kind") == "policy_violation": + session.events.append({"route": "goal_policy_violation", "step": observation.step_idx, + "reason": result["policy_violation"]}) + return FailAction(reason="policy_violation: " + result["policy_violation"]) + if self.empty_stages >= 3: + session.events.append({"route": "goal_stalled", "step": observation.step_idx, + "reason": "three_consecutive_stages_without_an_action", + "blocking_reason": result.get("failure_kind", "")}) + return FailAction(reason="three_consecutive_stages_without_an_action") + raise ObservationRequired("stage_verification_returned") + + def act(self, session, observation): + tree = session._read_tree(observation) + if tree: + tree = foreground_tree(tree) + screenshot = Path(observation.screenshot.path) + targets = session.components.target_builder.build( + tree, f"t{observation.step_idx}", 128, infer_button_labels=True, include_sliders=True, + context=" ".join(str(v) for k, v in (self.memory.active_stage or {}).items() + if k in {"goal", "completion_criteria", "method", "guidance"} + ) or session.instruction, filter_occluded=True, + ) if tree else [] + targets = session.components.semantic_enricher( + targets, tree, (observation.screenshot.width, observation.screenshot.height), + session._read_browser_snapshot(observation)) + self._receive(session, observation, targets, tree) + if self.end_reason: + return self._finish(session, observation) + if session.done_probe is not None: + result = resolve_done_probe(session, observation, tree) + if result is not None: + return result + raise ObservationRequired("terminal_rejected") + if self.memory.active_stage is None: + started = time.perf_counter() + try: + plan = session.components.planner( + session.instruction, screenshot, targets, session.recent, tree or "", + requirements=session.requirements, remaining_steps=observation.budget.steps_remaining, + memory=self.memory, remaining_seconds=observation.budget.wall_seconds_remaining) + except ModelResponseError as error: + session.events.append({"route": "planner_output_invalid", "step": observation.step_idx, + "reason": str(error)}) + return FailAction(reason="planner_output_invalid") + session.events.append({"route": "planner", "step": observation.step_idx, + "mode": "goal", "plan": plan, + "seconds": time.perf_counter() - started}) + if plan["done"]: + result = handle_done(session, plan, observation, tree) + if result is not None: + return result + raise ObservationRequired("terminal_rejected") + plan["max_actions"] = min(plan["max_actions"], self.max_actions) + self.memory.start(plan, observation.step_idx) + self._emit_state(session, observation.step_idx) + raise ObservationRequired("planner_returned") + stage = self.memory.active_stage + document = stage.get("target_document") + stage["document_recovery"] = bool(document and _document_mismatch(document, tree, targets)) + app = stage.get("target_application") + stage["application_recovery"] = bool(app and application_mismatch(app, tree)) + if stage["route"] == "cli": + action = CliAction(command=stage["command"], timeout_s=session.config.cli_timeout_seconds, + want_observation="screenshot") + self.pending = {"kind": "cli", "operation": "cli", "target": stage["method"], + "action": action.to_dict(), "state": _state_key(targets), + "before_screenshot": observation.screenshot.path} + session.events.append({"route": "cli", "step": observation.step_idx, + "stage_id": stage["id"], "action": action.to_dict()}) + return action + return self._gui(session, observation, targets, tree, screenshot) + + def _gui(self, session, observation, targets, tree, screenshot): + context = execution_context(self.memory) + stage = self.memory.active_stage + state = _loop_state(targets, tree) + work = stage["work_item_id"] + cached = self.grounded_states.get((work, state)) + if cached and (cached[1] != stage["id"] or _visual_changed(cached[0], screenshot)): + self.grounded_states.pop((work, state)) + session.events.append({"route": "recovery_cache_invalidated", "step": observation.step_idx, + "stage_id": stage["id"], "reason": "scene_or_subgoal_changed"}) + if self.visual and not self.interaction.reusable_visual(self.visual_frame, state, screenshot): + self.visual, self.visual_frame = [], None + self.grounded_states.pop((work, state), None) + session.events.append({"route": "recovery_invalidated", "step": observation.step_idx, + "stage_id": stage["id"], "reason": "last_action_invalidated_coordinates" + if self.interaction.invalidate_visual else "task_scene_changed"}) + recent = [a for a in self.attempts if a["work"] == work][-8:] + cycle_keys = _cycle_actions(recent, state) + cycle = bool(cycle_keys) + blocked = {key[2] for key, count in self.failed_actions.items() + if key[:2] == (work, state) and count >= 2} + if cycle: + blocked.update(cycle_keys) + for key in blocked: + failure_key = (work, state, key) + self.failed_actions[failure_key] = max(2, self.failed_actions.get(failure_key, 0)) + for key in blocked: + constraint = {"work_item_id": work, "window": list(state[:2]), "action_key": list(key), + "reason": "Repeated action/cycle in the same observed controls. Try a different " + "interaction; unrelated animation or new facts do not clear it."} + if constraint not in self.memory.execution_constraints: + self.memory.execution_constraints.append(constraint) + context["avoid_actions"] = [list(key) for key in blocked] + semantic_missing = (not any("gnome-shell" not in t.parent_context for t in targets) + or bool(portal_needs_grounding(tree))) + space = session.components.candidate_builder.build( + targets, 96, action_inputs=[], context=" ".join( + context[k] for k in ("goal", "completion_criteria", "method", "guidance"))) + space = goal_candidates(space, targets, tree, context["inputs"], + (observation.screenshot.width, observation.screenshot.height)) + if self.visual: + space.target_heads["visual"] = self.visual + if (work, state) in self.grounded_states: + space.controls.pop("visual_fallback", None) + for operation in space.target_heads: + space.target_heads[operation] = [c for c in space.target_heads[operation] + if _action_key(c, targets) not in blocked] + space.controls = {k: c for k, c in space.controls.items() if _action_key(c, targets) not in blocked} + context.update(self.interaction.finalize(space, targets, stage, self.memory, tree, + observation_ref=f"{stage['id']}:O{observation.step_idx}")) + control = self.interaction.control + if self.recovery_stage != (stage["id"], json.loads(control["scope_key"])[2]): + self.recovery_stage = (stage["id"], json.loads(control["scope_key"])[2]) + self.recovery_calls, self.recovery_seconds = 0, 0.0 + context["grounding_exhausted"] = ( + self.recovery_calls >= session.config.goal_recovery_max_calls + or self.recovery_seconds >= session.config.goal_recovery_max_seconds + or control.get("ground_calls", 0) >= session.config.goal_recovery_max_calls + or control.get("ground_seconds", 0) >= session.config.goal_recovery_max_seconds) + if context["grounding_exhausted"]: + space.controls.pop("visual_fallback", None) + self.memory.block("capability_exhausted", control["scope_key"], + "No further visual grounding in this unchanged input scope; " + "use offered observed actions or return for another route.", + f"{stage['id']}:O{observation.step_idx}") + context["active_blocker"] = self.memory.active_blocker + stage["interaction"] = self.interaction.snapshot() + missing = context["unbound_inputs"] and not any(c.source == "goal_input" + or c.metadata.get("commit_action") for c in space.candidates()) + need_grounding = ((semantic_missing or self.no_effect >= 2 or cycle or missing) + and not self.visual and not context["grounding_exhausted"] + and (work, state) not in self.grounded_states) + if need_grounding and session.components.grounder is not None: + if not self._ground(session, observation, screenshot, context, state): + return self._finish(session, observation) + if self.visual: + raise ObservationRequired("local_visual_recovery_returned") + if missing and not self.visual and stage["actions"] >= 4: + self.end_reason = "capability_missing: " + ", ".join(context["unbound_inputs"]) + return self._finish(session, observation) + if context["conflicting_fields"] and not any(c.source == "goal_input" for c in space.candidates()): + self.end_reason = "stage_input_conflict: " + ", ".join(context["conflicting_fields"]) + return self._finish(session, observation) + if (stage["purpose"] == "produce" and stage["actions"] >= 4 + and context["missing_required_fields"] and not context["next_intent"]): + self.end_reason = "stage_inputs_missing: " + ", ".join(context["missing_required_fields"]) + return self._finish(session, observation) + choice_context = self.interaction.choice_context(context) + current_goal = choice_context["objective"] + ranking = session.components.selector.rank( + stage_goal=current_goal, intent=json.dumps(choice_context, ensure_ascii=False), + active_window=ET.fromstring(tree).get("foreground_title", "") if tree else "", + targets=targets, action_space=space, + recent_actions=session.recent[-8:], screenshot=screenshot) + winner = ranking.winner + operation = winner.operation if winner else "return_to_planner" + target = next((t for t in targets if winner and t.id == winner.target_id), None) + action = session.components.compile_gui(winner, target) if winner else None + session.events.append({"route": "gui", "step": observation.step_idx, "mode": "goal", + "selector_backend": session.config.fast_backend, + "stage_id": self.memory.active_stage["id"], + "plan": {"stage_goal": context["goal"], "current_goal": current_goal}, + "input_frontier": {k: context[k] for k in ( + "next_intent", "missing_required_fields", "unbound_inputs", + "conflicting_fields", "satisfied_inputs")}, + "recovery_scope": { + k: context[k] for k in ("blocking_overlay", "overlay_bounds", + "document_recovery", "application_recovery") + if k in context}, + "candidates": [c.compact() for c in space.candidates()], + "winner": winner.compact() if winner else None, + "avoid_actions": context["avoid_actions"], + "execution_control": context["execution_control"], + "ranking": ranking_dict(ranking), + "action": action.to_dict() if action else None}) + if operation == "visual_fallback" and session.components.grounder is not None: + if (work, state) not in self.grounded_states: + if not self._ground(session, observation, screenshot, context, state): + return self._finish(session, observation) + if self.visual: + raise ObservationRequired("visual_grounder_returned") + if action is None: + self.end_reason = "stage_done_requested" if operation == "done" else "returned_to_planner" + return self._finish(session, observation) + self.pending = {"kind": action.kind, "operation": operation, + "target": winner.description, "action": action.to_dict(), + "state": _state_key(targets), "loop_state": state, + "candidate_key": _action_key(winner, targets), + "before_screenshot": observation.screenshot.path, + "expected_text": winner.metadata.get("text"), + "input_mode": winner.metadata.get("input_mode"), + "target_semantic": target.semantic_key if target else None, + "field_label": winner.metadata.get("field_label"), + "navigation_key": _navigation_key(winner, target, work, state), + "visual_action": operation == "scroll" or winner.source == "visual_grounder" + or target is not None and target.role == "canvas", + "fraction": winner.metadata.get("fraction")} + self.pending["overlay_bounds"] = context.get("overlay_bounds") + self.pending.update(control_scope=context["execution_control"]["scope_key"], + control_state=self.interaction.content_state) + self.pending.update({k: winner.metadata.get(k) for k in ( + "interaction_binding", "commit_action", "key_sequence", "execution_family")}) + self.pending["effect_bounds"] = winner.metadata.get("visual_bounds") or ( + [target.x, target.y, target.width, target.height] if target else None) + return action + + def _ground(self, session, observation, screenshot, context, state): + control = self.interaction.control + if (self.recovery_calls >= session.config.goal_recovery_max_calls + or self.recovery_seconds >= session.config.goal_recovery_max_seconds + or control is not None and (control.get("ground_calls", 0) + >= session.config.goal_recovery_max_calls or control.get("ground_seconds", 0) + >= session.config.goal_recovery_max_seconds)): + self.end_reason = "capability_missing: visual grounding budget exhausted" + self.memory.block("capability_exhausted", control["scope_key"], self.end_reason, + f"{self.memory.active_stage['id']}:O{observation.step_idx}") + session.events.append({"route": "recovery_exhausted", "step": observation.step_idx, + "stage_id": self.memory.active_stage["id"], + "calls": self.recovery_calls, "seconds": round(self.recovery_seconds, 3)}) + return False + started = time.monotonic() + self.recovery_calls += 1 + self.grounded_states[(self.memory.active_stage["work_item_id"], state)] = ( + str(screenshot), self.memory.active_stage["id"]) + plan = {**self.interaction.choice_context(context), + "current_goal": context.get("next_intent") or context["completion_criteria"], + "stage_goal": context["completion_criteria"], + "action_proposals": [{"operation": "type_text", **v} for v in context["inputs"] + if not launcher_input(v) or state[0] == "GNOME Overview"]} + self.visual = session.components.grounder(session.instruction, plan, screenshot) + elapsed = time.monotonic() - started + self.recovery_seconds += elapsed + if control is not None: + control.update(ground_calls=control.get("ground_calls", 0) + 1, + ground_seconds=control.get("ground_seconds", 0.0) + elapsed) + self.visual_frame = {"step": observation.step_idx, "state": state, "screenshot": str(screenshot)} + # The new frame was located after the invalidating action, not before it. + self.interaction.invalidate_visual = False + session.events.append({"route": "recovery", "step": observation.step_idx, + "stage_id": self.memory.active_stage["id"], + "method": "visual_grounding", "candidate_count": len(self.visual), + "avoid_actions": context.get("avoid_actions", []), + "attempt": self.recovery_calls}) + return True diff --git a/src/tandem/runtime/session.py b/src/tandem/runtime/session.py index 460337c..63c796e 100644 --- a/src/tandem/runtime/session.py +++ b/src/tandem/runtime/session.py @@ -18,6 +18,7 @@ from ..models.ranking import ranking_dict from ..protocol.models import Action, CliAction, FailAction, GuiAction, Observation, WaitAction from .episode import handle_done, resolve_done_probe, verify_pending +from .goal_stage import GoalExecution from .stages import StageExecution, recovery_scopes from .stopping import NO_ACTION_LIMIT, budget_exhausted @@ -30,6 +31,7 @@ def __init__(self, components: Components, *, config: SessionConfig | None = Non self.begin(requirements=[]) # 构造即重置,但不触发 requirements 抽取 def begin(self, instruction: str = "", requirements: list | None = None) -> None: + self.goal = None self.instruction = instruction # requirements=None(真实 run)时走组件抽取(旧 runtime 同款); # 显式 [](mock 链路)跳过 LLM 调用 @@ -45,9 +47,14 @@ def begin(self, instruction: str = "", requirements: list | None = None) -> None self.stage = ( StageExecution(self.config.stage_max_actions) if self.config.execution_mode == "stage" else None ) + if self.config.execution_mode == "goal": + self.goal = GoalExecution(self.components.memory_factory(instruction, requirements), + self.config.stage_max_actions, self.components.interaction_factory()) # ------------------------------------------------------------------ 主入口 def act(self, observation: Observation) -> Action: + if self.goal is not None: + return self.goal.act(self, observation) tree = self._read_tree(observation) if self.pending is not None: verify_pending(self, observation, tree) diff --git a/src/tandem/verification/goal_verifier.py b/src/tandem/verification/goal_verifier.py new file mode 100644 index 0000000..ad1171e --- /dev/null +++ b/src/tandem/verification/goal_verifier.py @@ -0,0 +1,423 @@ +"""One requirement-bound semantic verification per ended goal stage, never per GUI action.""" + +import base64 +import json +from copy import deepcopy +from pathlib import Path + +from ..models.slow import ModelResponseError + +PROMPT = """Independently verify the ENDED desktop subtask against its fixed work unit and ORIGINAL +requirements. Do not judge each intermediate action, demand later pages earlier, or trust the actor's +printed success. Return only JSON: +{stage_outcome:"met|not_met|uncertain",unit_met:false,delivery_state:"none|implemented|saved|verified", +summary:"specific observed result",remaining:["concrete missing part"],evidence_refs:["supplied id"], +facts:[{key:"registered or concrete reusable key",value:"exact content including numbers/units", +status:"confirmed|uncertain",scope:"task|stage",source_ref:"actual file/page/location", +evidence_refs:["supplied id"],complete:true}], +materials:[{source_ref:"actual file/url",covered_parts:["page/section"], +summary:"reusable source content",evidence_refs:["supplied id"]}], +requirement_checks:[{id:"R1",status:"met|not_met|uncertain",evidence_refs:["supplied id"]}], +artifacts:[{target_path:"actual final path",working_path:"actual working path",save_state:"modified|saved", +evidence_refs:["supplied id"]}],invalidated_work_item_ids:[], +milestones:[{key:"stable target/section identifier",summary:"actual delivered increment", +evidence_refs:["current supplied id"]}],policy_violation:"", +progress_kind:"none|navigation|source|content|save|inspection",observed_errors:[], +unit_assessment:{work_item_id:"exact fixed id",status:"met|not_met|uncertain", +evidence_refs:[],remaining:[]}}. +For purpose=prepare, OMIT unit_assessment: the host records not_assessed without asking you to judge +the delivery unit. Never accept a delivery unit during preparation. +Use [] when no update is established. Reference only the supplied evidence IDs. Preserve original +requirements and source facts. Extract essential facts NOW from real observed material/CLI output so +later stages do not reread it; unobserved contents remain unknown. Preparation uses delivery_state=none. +Use the exact facts_needed key when observed information answers that key. Keep partial source coverage +in materials even if not all required facts have been acquired. Opening a reference image, a contact +sheet, an audit or a script +that reports failure is not delivery progress. A finished stage need not satisfy a whole requirement. +unit_met=true ONLY when the full fixed work unit is satisfied; for delivery it requires verified level +and concrete evidence. Saved requires actual persistence to the target, not a working copy or title. +File existence/compilation does not prove content or function. Do not invent a demand to repeat every +successful intermediate action. Mark unmet/uncertain criteria explicitly and preserve actual progress. +Later contradictions invalidate affected units; do not invalidate old work from an uncertain screenshot +alone. Do not read evaluators, hidden grading rules, answers or rewards. Be concise and evidence-based.""" + +PROMPT += """ +Check protected state explicitly: opening a reference must not destroy an unsaved task form/document. +Do not claim another tab, preserved fields or a prior save still exists without current supporting evidence. +When current evidence proves a form reset, edits lost or the wrong artifact replaced the working result, +return the affected delivery work_item_id in invalidated_work_item_ids, even in a prepare stage. +Keep stable source facts, but withdraw stale UI/delivery milestones. A temporary working copy is not the +user's final deliverable: artifacts.target_path names that deliverable, working_path names the actual copy. +When publishing/reopening a previous working copy at its final path, include replaces_working_path with +the previous copy's exact path so its pending publication can be retired. +For file opening, inspect the resulting document instead of demanding its now-closed path entry. +A pending Open/Import confirmation is not a completed file handoff, unless a preview alone was requested. +""" + +PROMPT += ''' +delivery_state describes ONLY the CURRENT stage's new delivery, never an artifact's historical level. +For navigation/source/no new work use delivery_state=none; the host retains earlier saved/verified state. +Registered source-key updates belong to purpose=prepare acquisition stages, not form-state snapshots +during editing. Keep current form values in stage-scope field-specific notes, never overwrite a source key. +scope=task means reusable SOURCE CONTENT, not whole-task completion. scope=stage is only transient UI +state. A confirmed paragraph or data field is task-scope even while the rest of its source is unread. +Not seeing a source again does not contradict its recorded facts. Retain confirmed source values; +change one only with actual contradictory evidence and a nonempty correction_reason on that fact. +Use the precise registered source key when it is fully answered; store partial sections as separate +reusable facts/materials. Do not mark an entire packet confirmed when required fields are missing. +remaining must describe THIS stage's unmet exit conditions, not a list of every later task obligation. +Only mark unit_met when the original full unit is satisfied. Historical registered evidence may support +source facts; stage completion must also cite at least one CURRENT evidence ID. +Use the request's response_template for the mandatory keys and exact work_item_id; populate it with +observed evidence, not empty defaults. facts[].value is ALWAYS a JSON STRING, including numbers, dates, +lists and tables: use "25", not 25, and readable text, not an object or array. Optional facts[].entries +maps string keys to STRING values; store numbers as strings there too. A stage purpose=prepare is +source acquisition only even when attached to a delivery unit; it cannot finish that delivery unit. +For lists/tables/options, include entries:{"stable item/row/position":"observed value"} and complete:false +until the requested coverage is actually observed. New entries extend the old collection; conflicting +entries or scalar changes require current evidence and correction_reason. Never label a partial list +as a complete registered dependency. Temporary UI state is not required source content. +This request contains selected evidence, not the whole execution log. output_omitted_chars marks an +explicit excerpt: do not claim to have checked omitted contents or a complete source packet. Extract +only supported fields; ask for the missing section rather than another full-document audit. +For prepare, answer fact_targets using their EXACT registered keys and record partial coverage; +unit_met=false, delivery_state=none, and do not assess unrelated delivery requirements. +For produce/save/verify, check only the current stage exit condition and its fixed delivery unit. +Record partial implementation/saving without demanding all future task units. Return only memory deltas, +not a rewrite of existing facts/materials, and no new optional research or inspection requirements. +Missing source fields do not invalidate an independently useful, correctly implemented partial stage. +If that bounded exit condition is met, report stage_outcome=met with unit_met=false and the observed +implemented/saved level. Keep the full unit incomplete and preserve its missing values for later work. +Incomplete prose facts may be appended across stages: complete:false preserves earlier sections without +claiming a complete source. Mark complete:true only when the exact registered field/section is fully answered. +Never replace a known inventory with the contents of a different document under the same scalar key; +prefer document/field-specific facts and preserve source_ref. Do not guess identity or ownership. +For a real delivered increment, record a milestone with a stable artifact/section/field-group key. +Reuse the same key from fixed_work_item.milestones for the same increment; never key by stage ID, +rewording, research, audit files, navigation, or opening a source. New source facts are not delivery. +milestones:[] when nothing was actually implemented, saved or independently checked. +CLI actor assertions, ZIP/XML validity and marker strings establish neither visual fidelity nor working +animations. They can support saving only. Current GUI inspection must demonstrate visual requirements; +a blocking dialog is not evidence that the underlying content was checked. Do not weaken failed checks. +Respect ORIGINAL interaction restrictions. Filling website fields is not permission to edit the web +application's source, inject reload scripts, add missing options or remove validation. If the actions +violate these boundaries, return policy_violation with the specific observed action and do not accept it. +Planner method/guidance and invented workflow preferences are NOT user policy. Wrong application, +wrong shortcut, wrong navigation and incomplete work are recoverable stage errors, not policy violations. +For a genuine violation also return policy_constraint and policy_evidence_refs. policy_constraint must +be an EXACT quote of an ORIGINAL explicit prohibition, or one of website_source_tampering, +evaluator_access, credential_access for the host's integrity boundaries. policy_evidence_refs must cite +CURRENT actions demonstrating the violation, not an old screenshot or the planner's proposed method. +Do not infer tampering from ordinary CLI editing of a requested output document. Opening an unintended +application is not a prohibition violation unless the user explicitly forbids opening it. +Keep the existing working_path for the same target_path. When current evidence demonstrates a real +working-copy migration preserving prior increments, include replaces_working_path with the old exact +working path on that artifact. A fresh source copy or unrelated per-section file is not such a migration. +progress_kind and observed_errors are mandatory. For produce/save/verify also return unit_assessment +with the exact fixed work_item_id, status met|not_met|uncertain, evidence_refs and remaining arrays. +stage_outcome and remaining judge ONLY stage_acceptance. unit_assessment judges ONLY unit_acceptance, +NOT all sibling units or the overall requirement. The entire task is NOT being accepted here. +If a save/open unit is fully supported, mark that UNIT met even when sibling animation/content units +remain pending. Do not demand their contents in this unit. If unit_assessment.status=met, return +unit_met=true and delivery_state=verified with current evidence. Preparation omits unit_assessment. +Opening/maximizing windows, menus, scrolling, and dismissing dialogs are navigation: delivery_state=none, +unit_met=false,milestones:[], even if their bounded preparation stage succeeded. Source reading is source, +not content delivery. A saved partial artifact may be save progress without finishing its whole unit. +observed_errors is [{message:"EXACT error visible in an action receipt or current screenshot", +application:"observed application",command_name:"exact command if explicitly named, otherwise empty", +evidence_refs:["CURRENT id"]}]. Do not replace the error with remaining steps or invent errors. +Unknown command means the command was not recognized; extra arguments are not evidence of support. +Corrective advice belongs in remaining, never in observed_errors or confirmed source facts. +''' + + +def result_error(value): + """Precise contract failures; never invent evidence or coerce unsupported results.""" + for key, choices in {"stage_outcome": {"met", "not_met", "uncertain"}, + "delivery_state": {"none", "implemented", "saved", "verified"}}.items(): + if value.get(key) not in tuple(choices): + return key + " must be one of " + str(sorted(choices)) + for key, kind in {"unit_met": bool, "summary": str}.items(): + if type(value.get(key)) is not kind: + return key + " must be " + kind.__name__ + lists = ("remaining", "evidence_refs", "facts", "materials", "requirement_checks", "artifacts", + "invalidated_work_item_ids") + for key in lists: + if not isinstance(value.get(key), list): + return key + " must be an array (use [] for no updates)" + for key in ("remaining", "evidence_refs", "invalidated_work_item_ids"): + if any(not isinstance(x, str) for x in value[key]): + return key + " must contain only strings" + if value.get("progress_kind", "none") not in { + "none", "navigation", "source", "content", "save", "inspection"}: + return "progress_kind must describe observed navigation, source, content, save or inspection" + if not isinstance(value.get("observed_errors", []), list): + return "observed_errors must be an array" + for error in value.get("observed_errors", []): + if (not isinstance(error, dict) or any(not isinstance(error.get(k), str) for k in ( + "message", "application", "command_name")) or not error["message"].strip() + or not isinstance(error.get("evidence_refs"), list) + or any(not isinstance(ref, str) for ref in error["evidence_refs"])): + return "observed_errors need exact message, application, command_name and evidence_refs" + check = value.get("unit_assessment") + if "unit_assessment" in value and (not isinstance(check, dict) + or not isinstance(check.get("work_item_id"), str) + or check.get("status") not in {"met", "not_met", "uncertain", "not_assessed"} + or any(not isinstance(check.get(k), list) or any(not isinstance(x, str) for x in check[k]) + for k in ("remaining", "evidence_refs"))): + return "unit_assessment needs work_item_id, status, remaining and evidence_refs" + if not isinstance(value.get("policy_violation", ""), str): + return "policy_violation must be a string" + if not isinstance(value.get("policy_constraint", ""), str): + return "policy_constraint must be a string" + if (not isinstance(value.get("policy_evidence_refs", []), list) + or any(not isinstance(ref, str) for ref in value.get("policy_evidence_refs", []))): + return "policy_evidence_refs must be an array of strings" + if not isinstance(value.get("milestones", []), list): + return "milestones must be an array" + for record in value.get("milestones", []): + if (not isinstance(record, dict) or any(not isinstance(record.get(k), str) or not record[k].strip() + for k in ("key", "summary")) + or not isinstance(record.get("evidence_refs"), list) + or any(not isinstance(ref, str) for ref in record["evidence_refs"])): + return "milestones need a stable key, summary and evidence_refs array of strings" + fields = {"facts": ("key", "value", "source_ref"), "materials": ("source_ref", "summary"), + "requirement_checks": ("id",), "artifacts": ("target_path", "working_path")} + enums = {"facts": {"status": {"confirmed", "uncertain"}, "scope": {"task", "stage"}}, + "requirement_checks": {"status": {"met", "not_met", "uncertain"}}, + "artifacts": {"save_state": {"modified", "saved"}}} + for collection, strings in fields.items(): + for index, record in enumerate(value[collection]): + path = f"{collection}[{index}]" + if not isinstance(record, dict): + return path + " must be an object" + for key in strings: + if not isinstance(record.get(key), str): + return path + "." + key + " must be a string" + arrays = ("evidence_refs", "covered_parts") if collection == "materials" else ("evidence_refs",) + for key in arrays: + if not isinstance(record.get(key), list) or any(not isinstance(x, str) for x in record[key]): + return path + "." + key + " must be an array of strings" + for key, choices in enums.get(collection, {}).items(): + if record.get(key) not in tuple(choices): + return path + "." + key + " must be one of " + str(sorted(choices)) + if collection == "facts": + if "complete" in record and type(record["complete"]) is not bool: + return path + ".complete must be a boolean" + if "entries" in record and (not isinstance(record["entries"], dict) or any( + not isinstance(k, str) or not isinstance(v, str) + for k, v in record["entries"].items())): + return path + ".entries must map strings to strings" + return "" + + +def valid_result(value): + return not result_error(value) + + +def _excerpt(text, limit): + if len(text) <= limit: + return text + return text[:limit // 2] + "\n[... omitted source content; not inspected ...]\n" + text[-limit // 2:] + + +def _actions(history, output_budget): + """Keep receipts and semantic deltas, not repeated snapshot IDs/ancestry/whole trees.""" + actions = [] + previous = {} + output_records = max(1, sum(bool(r.get("full_output_ref") or r.get("stdout") or r.get("stderr")) + for r in history)) + limit = max(1000, output_budget // output_records) + for record in history: + action = {k: record[k] for k in ("evidence_ref", "kind", "operation", "target", "returncode", + "observed_change", "relevant_change", "target_effect", + "full_output_ref") if k in record} + if record.get("kind") != "cli" and record.get("action"): + action["action"] = record["action"] + output = dict(record) + if record.get("full_output_ref"): + output.update(json.loads(Path(record["full_output_ref"]).read_text(encoding="utf-8"))) + omitted = {} + for key in ("stdout", "stderr"): + text = output.get(key, "") + if text: + action[key] = _excerpt(text, limit) + if len(text) > limit: + omitted[key] = len(text) - limit + if omitted: + action["output_omitted_chars"] = omitted + changes, current = [], {} + for control in record.get("observed_controls", []): + key = (control.get("role"), control.get("name"), control.get("x"), control.get("y")) + value = {k: control[k] for k in ("role", "name", "value", "focused", "checked", "selected") + if k in control} + current[key] = value + if value != previous.get(key): + changes.append(value) + previous = current + if changes: + action["control_changes"] = changes + actions.append(action) + return actions + + +def _failure_result(memory, history, end_reason, evidence, error=None): + """No semantic success claim from a known error or an entirely ineffective local attempt.""" + pure_no_effect = (history and all(h.get("kind") == "gui" + and not h.get("relevant_change", h.get("observed_change")) + for h in history)) + skip = (error is not None or end_reason == "execution_error" or end_reason.startswith(( + "stage_inputs_missing:", "stage_input_conflict:", "capability_missing:")) + or not history and end_reason != "stage_done_requested" + or pure_no_effect and not any(h.get("operation") == "type_text" for h in history) + and memory.active_stage["purpose"] not in {"save", "verify"}) + if not skip: + return None + error = error or next((h.get("stderr") or h.get("stdout") for h in reversed(history) + if h.get("returncode")), "") + return {"stage_outcome": "uncertain", "unit_met": False, "delivery_state": "none", + "summary": "No semantic acceptance: " + end_reason + + (". Observed error: " + error[-1200:] if error else ""), + "remaining": [end_reason + ": inspect partial effects and change method; do not repeat it."], + "evidence_refs": list(evidence)[-1:], "facts": [], "materials": [], + "requirement_checks": [], "artifacts": [], "invalidated_work_item_ids": [], + "check_mode": "local_failure"} + + +def verify_goal(client, instruction, screenshot, *, memory, history, end_reason, evidence): + local = _failure_result(memory, history, end_reason, evidence) + if local is not None: + local["failure_kind"] = end_reason + return local + stage = memory.active_stage + work = memory.work_items[stage["work_item_id"]] + targets = stage.get("fact_targets", work["facts_needed"]) + keys = set(stage["input_fact_keys"]) | {f["key"] for f in targets} + budget = client.config.stage_verify_max_chars + actions = _actions(history, budget // 2) + preparing = stage["purpose"] == "prepare" + template = {"stage_outcome": "uncertain", "unit_met": False, "delivery_state": "none", + "summary": "Describe current observed evidence", "remaining": [], "evidence_refs": [], + "facts": [], "materials": [], "requirement_checks": [], "artifacts": [], + "invalidated_work_item_ids": [], "milestones": [], "policy_violation": "", + "progress_kind": "none", "observed_errors": []} + if not preparing: + template["unit_assessment"] = {"work_item_id": work["id"], "status": "uncertain", + "evidence_refs": [], "remaining": []} + context = {"task": instruction, + "response_template": template, + "stage_acceptance": {"criterion": stage["completion_criteria"], "scope": "ended_stage_only"}, + "unit_acceptance": ({"work_item_id": work["id"], "scope": "not_assessed_during_prepare"} + if preparing else {"work_item_id": work["id"], + "criterion": work["completion_criteria"], + "scope": "this_fixed_unit_only_not_all_original_requirements"}), + "sibling_units_not_current_acceptance": [{"id": w["id"], "goal": w["goal"]} + for w in memory.work_items.values() if w["kind"] == "deliver" and w["id"] != work["id"] + and w["requirement_ids"] == work["requirement_ids"]], + "stage": {k: v for k, v in stage.items() if k != "command"}, + "command_excerpt": _excerpt(stage.get("command", ""), 3000), + "fixed_work_item": {**work, "facts_needed": targets}, + "requirements": [{k: r[k] for k in ("id", "requirement", "kind")} + for r in memory.requirements if r["id"] in work["requirement_ids"] + or r["kind"] == "constraint"], + "known_facts": [f for f in memory.facts.values() if f["key"] in keys], + "end_reason": end_reason, "actual_actions": actions, "evidence_ids": list(evidence), + "historical_evidence_ids": sorted(memory.known_evidence)} + text = json.dumps(context, ensure_ascii=False) + if len(text) > budget: + # Receipt deltas stay intact. Explicitly excerpt source text, never silently claim full coverage. + for index, original in enumerate(context["known_facts"]): + fact = {k: v for k, v in original.items() if k != "entries"} + fact["value"] = _excerpt(fact["value"], max(1000, budget // (2 * max(1, len(keys))))) + if fact["value"] != original["value"]: + fact["value_excerpt_only"] = True + context["known_facts"][index] = fact + text = json.dumps(context, ensure_ascii=False) + content = [{"type": "text", "text": text}] + source_frames = [h["screenshot"] for h in history if h.get("source_frame")] + paths = source_frames[-5:] if stage["purpose"] == "prepare" else ( + [history[0]["screenshot"]] if history else []) + frames = [Path(p) for p in dict.fromkeys(paths + [str(screenshot)])] + for path in frames: + content.extend([{"type": "text", "text": "Observed screenshot: " + str(path)}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + + base64.b64encode(path.read_bytes()).decode("ascii")}}]) + messages = [{"role": "system", "content": PROMPT}, {"role": "user", "content": content}] + + def validate(value): + error = result_error(value) + if not error: + check = value.get("unit_assessment", {}) + required = ("progress_kind", "observed_errors") + (() if preparing else ("unit_assessment",)) + if not all(key in value for key in required): + error = "include mandatory fields: " + ", ".join(required) + elif preparing and (value["unit_met"] or value["delivery_state"] != "none" + or value.get("milestones")): + error = "prepare cannot accept a delivery unit or claim delivery progress" + elif not preparing and check.get("work_item_id") != work["id"]: + error = "unit_assessment.work_item_id must be " + work["id"] + elif not preparing and (check.get("status") == "met") != value["unit_met"]: + error = "unit_met must agree with this unit_assessment, not whole-task completion" + elif value["unit_met"] and value["delivery_state"] != "verified": + error = "a met delivery unit requires verified evidence for its own fixed criterion" + elif value["progress_kind"] in {"navigation", "source", "none"} and ( + value["unit_met"] or value["delivery_state"] != "none" or value.get("milestones")): + if value["unit_met"] or value.get("milestones"): + error = "navigation/source/no delivery cannot claim units or delivery milestones" + else: + value["historical_delivery_claim"] = value["delivery_state"] + value.update(delivery_state="none", artifacts=[]) + if not error: + allowed = set(evidence) | memory.known_evidence + records = [value, *value["facts"], *value["materials"], + *value["artifacts"], *value["requirement_checks"], *value.get("milestones", []), + *value.get("observed_errors", [])] + if not preparing: + records.append(value["unit_assessment"]) + unknown = sorted({ref for r in records for ref in r["evidence_refs"] if ref not in allowed}) + if unknown: + error = "evidence_refs contain unregistered IDs: " + json.dumps(unknown) + elif value["stage_outcome"] == "met" and not set(value["evidence_refs"]) & set(evidence): + error = "a met stage must cite at least one CURRENT evidence ID" + elif any(not set(item["evidence_refs"]) or not set(item["evidence_refs"]) <= set(evidence) + for item in value["observed_errors"]): + error = "observed_errors must cite CURRENT observed evidence" + elif value["unit_met"] and not set(value["unit_assessment"]["evidence_refs"]) & set(evidence): + error = "a met unit must cite CURRENT evidence for its fixed criterion" + if error: + messages.append({"role": "user", "content": + "Invalid stage verification JSON: " + error + ". Return the COMPLETE corrected object " + "with every mandatory key, without changing the observed facts or inventing evidence."}) + return not error + + try: + result = client.json_request(messages, validate, role="verify_stage") + except ModelResponseError as error: + result = _failure_result(memory, history, "verification_output_invalid", evidence, str(error)) + return {**result, "failure_kind": "verification_output_invalid"} + result = deepcopy(result) + if preparing: + result["unit_assessment"] = {"work_item_id": work["id"], "status": "not_assessed", + "evidence_refs": [], "remaining": []} + if result.get("progress_kind") in {"navigation", "source", "none"}: + result.update(delivery_state="none", unit_met=False, milestones=[]) + if stage["route"] == "cli" and stage["purpose"] in {"produce", "save"}: + if result["delivery_state"] == "verified": + result.update(unit_met=False, delivery_state="saved", + acceptance_limited="cli_save_not_gui_acceptance") + if result.get("policy_violation"): + constraint = result.get("policy_constraint", "").strip() + refs = set(result.get("policy_evidence_refs", [])) + current_actions = {h["evidence_ref"] for h in history if h.get("evidence_ref") in evidence} + original = (len(constraint) >= 12 and constraint in instruction and any( + token in constraint.casefold() for token in ("not ", "never ", "forbid", "禁止", "不得"))) + integrity = constraint in {"website_source_tampering", "evaluator_access", "credential_access"} + fatal = bool(refs and refs <= current_actions and (original or integrity)) + result.update(stage_outcome="not_met", unit_met=False, delivery_state="none", milestones=[], + artifacts=[], requirement_checks=[], remaining=[result["policy_violation"]], + failure_kind="policy_violation" if fatal else "recoverable_stage_error") + if fatal: + result.update(facts=[], materials=[]) + result.update(check_mode="source" if stage["purpose"] == "prepare" else "delivery", + context_chars=len(text), image_count=len(frames)) + return result diff --git a/src/tandem/verification/verifier.py b/src/tandem/verification/verifier.py index 18c817d..3e44ba6 100644 --- a/src/tandem/verification/verifier.py +++ b/src/tandem/verification/verifier.py @@ -37,6 +37,7 @@ def verify_observation( requirements=None, output_files=None, execution_result=None, + task_memory=None, ): context = { "task": instruction, @@ -50,6 +51,8 @@ def verify_observation( } if execution_result is not None: context["execution_result"] = execution_result + if task_memory is not None: + context["verified_task_memory"] = task_memory content = [{"type": "text", "text": json.dumps(context, ensure_ascii=False)}] for label, path in (("BEFORE", previous), ("CURRENT", screenshot)): if path is not None: @@ -107,6 +110,13 @@ def verify_observation( "A CLI read-only inspection can be achieved without a GUI change; preserve state_changed=false " "in that case. Report contradictory command failures instead of assuming the intended edit happened." ) + if task_memory is not None: + messages[0]["content"] += ( + " For whole-task verification, use supplied independently verified stage evidence and " + "confirmed source facts as historical evidence; do not require every prior page or source " + "to appear simultaneously in the current screenshot. The ledger is not infallible: current " + "contradictions override it. A pending publication or unsaved working copy is not delivery." + ) result = client.json_request( messages, lambda value: ( @@ -147,6 +157,9 @@ def verify_observation( not exists for exists in (output_files or {}).values() ): result.update(task_status="incomplete", outcome="not_achieved") + if task_memory and any(a.get("delivery_pending") for a in task_memory["artifacts"]): + result.update(task_status="incomplete", outcome="not_achieved", + evidence=result["evidence"] + " Working-copy publication remains pending.") return result diff --git a/src/tandem_eval/__main__.py b/src/tandem_eval/__main__.py index b4b30a8..8dad50e 100644 --- a/src/tandem_eval/__main__.py +++ b/src/tandem_eval/__main__.py @@ -36,7 +36,7 @@ def main(argv: list[str] | None = None) -> int: run_cmd.add_argument("--allow-cli", choices=["true", "false"], default=None, help="harness 是否允许 CLI 路由") run_cmd.add_argument("--route-policy", choices=["auto", "gui", "cli"], default=None) - run_cmd.add_argument("--execution-mode", choices=["step", "stage"], default=None) + run_cmd.add_argument("--execution-mode", choices=["step", "stage", "goal"], default=None) run_cmd.add_argument("--stage-max-actions", type=int, default=None, help="阶段内动作数上限,达到后请求剩余工作的重规划") run_cmd.add_argument("--slow-model", default=None, metavar="NAME", diff --git a/src/tandem_eval/browser.py b/src/tandem_eval/browser.py index 358497c..4d8a41b 100644 --- a/src/tandem_eval/browser.py +++ b/src/tandem_eval/browser.py @@ -40,6 +40,8 @@ 'gridcell','combobox','textbox','searchbox','spinbutton','slider']); const selector = 'a[href],button,input,textarea,select,summary,[contenteditable="true"],' + [...roles].map(role => '[role="' + role + '"]').join(','); + const clickable = element => element.tabIndex >= 0 || typeof element.onclick === 'function' || + getComputedStyle(element).cursor === 'pointer'; const secretAutocomplete = new RegExp( '(^|\\s)(current-password|new-password|one-time-code|cc-number|' + 'cc-csc|cc-exp|cc-exp-month|cc-exp-year)(\\s|$)'); @@ -55,9 +57,9 @@ if (['button','submit','reset','image'].includes(element.type)) return 'button'; if (element.type === 'search') return 'searchbox'; if (element.type === 'number') return 'spinbutton'; - if (['text','email','url','tel'].includes(element.type)) return 'textbox'; + if (['text','email','url','tel','date'].includes(element.type)) return 'textbox'; } - return null; + return clickable(element) ? 'button' : null; }; const roleNames = {button:'push_button', link:'link', checkbox:'check_box', radio:'radio_button', switch:'switch', tab:'page_tab', menuitem:'menu_item', option:'option', gridcell:'table_cell', @@ -94,10 +96,27 @@ clone.querySelectorAll('input,textarea,select,[contenteditable="true"]').forEach(node => node.remove()); return norm(clone.innerText).slice(0, 240); }; + const itemContext = element => { + const row = element.closest('li,tr,[role="row"]'); + if (row) return contextText(row); + for (let node = element.parentElement, depth = 0; node && depth < 4; + node = node.parentElement, depth++) { + if (node.matches('body,html,form,dialog,[role="dialog"]')) break; + const text = contextText(node); + if (text.length > name(element).length + 8 && text.length < 240 && + node.querySelectorAll(selector).length <= 8) return text; + } + return contextText(element.closest('fieldset,form,dialog,[role="dialog"]')); + }; const elements = []; let rawCount = 0; - for (const element of document.querySelectorAll(selector)) { + for (const element of document.querySelectorAll(selector + ',div,span,li,td')) { rawCount++; + if (!element.matches(selector)) { + if (!clickable(element)) continue; + const parent = element.parentElement; + if (parent && (parent.closest(selector) || clickable(parent))) continue; + } if (!visible(element) || element.disabled || element.getAttribute('aria-disabled') === 'true') continue; if (element.type === 'password' || element.type === 'hidden' || element.type === 'file' || secretAutocomplete.test(element.getAttribute('autocomplete') || '')) continue; @@ -105,6 +124,8 @@ const cx = rect.left + rect.width / 2, cy = rect.top + rect.height / 2; if (rect.width < 2 || rect.height < 2 || cx < 0 || cy < 0 || cx >= innerWidth || cy >= innerHeight) continue; + const hit = document.elementFromPoint(cx, cy); + if (!hit || !element.contains(hit)) continue; // A visible background node may still be occluded. const domRole = roleOf(element); if (!domRole) continue; const hint = hintFor(rect); @@ -112,17 +133,22 @@ const editable = !element.readOnly && element.getAttribute('aria-readonly') !== 'true' && (['textbox','searchbox','spinbutton'].includes(domRole) || (domRole === 'combobox' && ['INPUT','TEXTAREA'].includes(element.tagName))); - const contextElement = element.closest('form,dialog,[role="dialog"],fieldset,li,tr,[role="row"]'); const item = { index: elements.length + 1, role: roleNames[domRole] || domRole, name: label, description: norm(element.getAttribute('aria-description') || element.getAttribute('title') || element.getAttribute('placeholder')), - value: 'value' in element ? String(element.value).slice(0, 200) : '', + value: element.tagName === 'SELECT' ? norm(element.selectedOptions[0]?.label) : + 'value' in element ? String(element.value).slice(0, 200) : '', + input_type: ['INPUT','SELECT'].includes(element.tagName) ? element.type : '', + focused: document.activeElement === element, + date_order: element.type === 'date' ? new Intl.DateTimeFormat(navigator.language) + .formatToParts(new Date(2001,10,22)).filter(p => ['year','month','day'].includes(p.type)) + .map(p => p.type) : [], x: originX + rect.left, y: originY + rect.top, width: rect.width, height: rect.height, scope: 'web_content', label_source: name(element) ? 'dom' : 'dom_nearby', - nearby_text: contextText(contextElement), + nearby_text: itemContext(element), operations: editable ? ['click','type_text'] : ['click'], source: 'dom' }; diff --git a/src/tandem_eval/config_schema.py b/src/tandem_eval/config_schema.py index e6d9ceb..6d696c1 100644 --- a/src/tandem_eval/config_schema.py +++ b/src/tandem_eval/config_schema.py @@ -70,8 +70,8 @@ "harness.fast_model.image_stage.read", ) -#: 快模型后端协议(决定用哪种排序传输:jev 的 /rank 还是 kev 的 /v1/systemone) -FAST_BACKENDS = ("jev", "kev") +#: 选择后端:jev /rank、kev /v1/systemone;planner 复用 slow_model,不需要独立 endpoint。 +FAST_BACKENDS = ("jev", "kev", "planner") #: 慢模型每个 API 条目必填的键 SLOW_API_KEYS = ("base_url", "model") @@ -175,7 +175,7 @@ def _validate_fast_models(config: dict) -> list[str]: endpoint = api.get("endpoint") endpoint_file = api.get("endpoint_file") # endpoint 允许留空 = 自动从发现文件读取最新地址(服务闲时部署,地址会变) - if _is_missing(endpoint) and _is_missing(endpoint_file): + if api.get("backend") != "planner" and _is_missing(endpoint) and _is_missing(endpoint_file): problems.append( f"harness.fast_model.apis.{name}.endpoint / .endpoint_file(至少给一个)" ) @@ -241,8 +241,8 @@ def validate(config: dict) -> list[str]: if not isinstance(execution, dict): problems.append("harness.execution(应为对象)") else: - if execution.get("mode", "step") not in {"step", "stage"}: - problems.append("harness.execution.mode(应为 step / stage)") + if execution.get("mode", "step") not in {"step", "stage", "goal"}: + problems.append("harness.execution.mode(应为 step / stage / goal)") limit = execution.get("stage_max_actions", 24) if type(limit) is not int or limit < 1: problems.append("harness.execution.stage_max_actions(应为正整数)") diff --git a/src/tandem_eval/entry.py b/src/tandem_eval/entry.py index 7406ec7..e593304 100644 --- a/src/tandem_eval/entry.py +++ b/src/tandem_eval/entry.py @@ -103,8 +103,15 @@ def __init__(self, *, stage_mode=False): self.stage_mode = stage_mode def __call__(self, instruction, screenshot, targets, recent, tree, - verified_focus, requirements, remaining_steps): + verified_focus=None, requirements=None, remaining_steps=None, *, memory=None, + remaining_seconds=None): self.calls += 1 + if memory is not None and self.calls == 1: + return {"done": False, "work_item_id": memory.ready_items()[0]["id"], + "route": "gui", "purpose": "produce", "goal": "mock click", + "completion_criteria": "mock receipt", "method": "mock click", + "inputs": [], "input_fact_keys": [], "guidance": "", "max_actions": 1, + "command": ""} if self.calls == 1 and targets: target = targets[0] plan = { @@ -146,19 +153,32 @@ def rank(self, **kwargs): ) -def mock_components(*, stage_mode=False) -> Components: +def mock_components(*, stage_mode=False, goal_mode=False) -> Components: from tandem.gui.candidates import ActionCandidateBuilder from tandem.gui.compile import gui_action from tandem.gui.proposals import focused_candidates from tandem.gui.semantics import enrich_targets from tandem.gui.targets import A11yTargetBuilder + from tandem.memory.state import TaskMemory + + def goal_verifier(*args, **kwargs): + return {"stage_outcome": "met", "unit_met": True, "delivery_state": "verified", + "summary": "mock evidence only", "remaining": [], + "evidence_refs": list(kwargs["evidence"]), "facts": [], "materials": [], + "requirement_checks": [], "artifacts": [], "invalidated_work_item_ids": []} + + def requirements(instruction): + return [{"id": "R1", "requirement": instruction, "kind": "delivery", "units": [instruction], + "unit_fact_keys": [[]], "facts_needed": [], "output_path": None}] if goal_mode else [] return Components( planner=_ScriptedPlanner(stage_mode=stage_mode), verifier=_ScriptedVerifier(), - requirements=lambda instruction: [], selector=_FirstCandidateSelector(), + requirements=requirements, selector=_FirstCandidateSelector(), target_builder=A11yTargetBuilder(), candidate_builder=ActionCandidateBuilder(), semantic_enricher=enrich_targets, focus_candidates=focused_candidates, grounder=None, compile_gui=gui_action, + memory_factory=TaskMemory if goal_mode else None, + stage_verifier=goal_verifier if goal_mode else None, ) @@ -166,14 +186,16 @@ def build_session(spec: dict, *, llm_tap: LlmTap | None = None) -> TandemSession config = SessionConfig(**spec.get("harness_config", {})) benchmark = spec.get("benchmark") if benchmark == "mock": - return TandemSession(mock_components(stage_mode=config.execution_mode == "stage"), config=config) + return TandemSession(mock_components(stage_mode=config.execution_mode == "stage", + goal_mode=config.execution_mode == "goal"), config=config) if benchmark in {"osworld1", "osworld2", "osworld21"}: from tandem.components import build_default from tandem.models.slow import SlowModelClient secrets = spec["secrets"] credentials = dict(secrets["slow_model"]) - endpoint = resolve_fast_model_endpoint(secrets.get("fast_model") or {}) + endpoint = ("" if config.fast_backend == "planner" else + resolve_fast_model_endpoint(secrets.get("fast_model") or {})) run_root = Path(spec["output_root"]) run_root.mkdir(parents=True, exist_ok=True) client = SlowModelClient(credentials, config, run_root / "slow_model_http.jsonl", diff --git a/src/tandem_eval/env.py b/src/tandem_eval/env.py index 38f5c5f..77b2bd3 100644 --- a/src/tandem_eval/env.py +++ b/src/tandem_eval/env.py @@ -7,10 +7,13 @@ from __future__ import annotations import hashlib +import io from dataclasses import dataclass, field from pathlib import Path from typing import Protocol +from PIL import Image + from tandem.protocol.models import ( AccessibilityTree, Action, @@ -30,6 +33,8 @@ def reset(self, task: dict) -> Observation: ... def execute(self, action: Action, *, budget: Budget, task: dict) -> tuple[Receipt, Observation]: ... + def observe(self, task: dict, *, budget: Budget, previous: Observation) -> Observation: ... + def fetch_task(self, task_id: str) -> dict: ... def export_outputs(self, paths: list[str], episode_dir) -> list[dict]: ... @@ -75,7 +80,9 @@ def fetch_task(self, task_id: str, domain: str = "") -> dict: def _screenshot_ref(self, task_id: str, step_idx: int) -> tuple[str, str]: key = f"{task_id}:{step_idx}" if key not in self.shots: - self.shots[key] = f"png-{key}".encode() + image = io.BytesIO() + Image.new("RGB", (1920, 1080), "white").save(image, format="PNG") + self.shots[key] = image.getvalue() digest = hashlib.sha256(self.shots[key]).hexdigest() if self.base_dir is not None: target = self.base_dir / "screenshots" / f"step_{step_idx:02d}.png" @@ -140,6 +147,9 @@ def execute(self, action: Action, *, budget: Budget, task: dict) -> tuple[Receip self.screenshot_counter += 1 return receipt, self._observation(task, step_idx, budget, receipt) + def observe(self, task, *, budget, previous): + return self._observation(task, previous.step_idx, budget, previous.last_receipt) + def export_outputs(self, paths: list[str], episode_dir) -> list[dict]: records = [] for index, path in enumerate(paths): diff --git a/src/tandem_eval/osworld.py b/src/tandem_eval/osworld.py index f8d3905..064889d 100644 --- a/src/tandem_eval/osworld.py +++ b/src/tandem_eval/osworld.py @@ -16,6 +16,7 @@ import struct import sys import time +import xml.etree.ElementTree as ET from collections.abc import Callable from pathlib import Path from typing import Any @@ -39,6 +40,20 @@ RECEIPT_TAIL = 12000 +_FOREGROUND_QUERY = """import json +from Xlib.display import Display +d=Display() +p=d.screen().root.get_full_property(d.intern_atom('_NET_ACTIVE_WINDOW'),0) +info={'foreground_title':'','foreground_class':[]} +if p is not None and int(p.value[0]): + w=d.create_resource_object('window',int(p.value[0])) + title=w.get_full_property(d.intern_atom('_NET_WM_NAME'),0) + info={'foreground_title':title.value.decode('utf-8','replace') if title else str(w.get_wm_name() or ''), + 'foreground_class':list(w.get_wm_class() or [])} +print(json.dumps(info)) +d.close() +""" + def png_size(data: bytes) -> tuple[int, int]: width, height = struct.unpack(">II", data[16:24]) @@ -47,8 +62,9 @@ def png_size(data: bytes) -> tuple[int, int]: def gui_code(action: GuiAction) -> str: params = action.parameters - if action.action_type in {"click", "double_click"}: - method = "click" if action.action_type == "click" else "doubleClick" + if action.action_type in {"click", "double_click", "right_click"}: + method = {"click": "click", "double_click": "doubleClick", "right_click": "rightClick"}[ + action.action_type] return f"pyautogui.{method}(x={params['x']}, y={params['y']})" if action.action_type == "hotkey": steps = [] @@ -66,7 +82,12 @@ def gui_code(action: GuiAction) -> str: steps.append("pyautogui.press('f2')") if params.get("input_mode", "replace") == "replace": steps.append("pyautogui.hotkey('ctrl', 'a')") - steps.append(f"pyautogui.write({params['text']!r})") + if params["text"].isascii(): + steps.append(f"pyautogui.write({params['text']!r}, interval=0.03)") + else: + steps.append("__import__('subprocess').run(['xclip','-selection','clipboard','-t','UTF8_STRING']," + f"input={params['text']!r}.encode('utf-8'),check=True)") + steps.append("pyautogui.hotkey('ctrl','v')") return "; ".join(steps) raise ValueError(f"不支持的 GUI 动作类型: {action.action_type}") @@ -126,6 +147,7 @@ def _bootstrap_runtime(self) -> None: os.environ["OSWORLD_FILE_BASE_URL"] = str(self.asset_root) self.env: Any = None self._step_seq = 0 + self._frame_seq = 0 self.last_screenshot: bytes = b"" self.last_a11y: str = "" self.last_browser_snapshot: dict | None = None @@ -182,13 +204,15 @@ def _observation(self, task: dict, step_idx: int, budget: Budget, receipt: Receipt | None) -> Observation: stage = self.run_root / "obs_stage" / str(task["id"]) stage.mkdir(parents=True, exist_ok=True) - shot_path = stage / f"step_{step_idx:02d}.png" + self._frame_seq += 1 + frame_name = f"step_{step_idx:02d}_obs_{self._frame_seq:04d}" + shot_path = stage / f"{frame_name}.png" shot_path.write_bytes(self.last_screenshot) a11y_path: str | None = None digest = node_count = None present = bool(self.last_a11y.strip()) if present: - a11y_path = str(stage / f"step_{step_idx:02d}.xml") + a11y_path = str(stage / f"{frame_name}.xml") Path(a11y_path).write_text(self.last_a11y, encoding="utf-8") digest = hashlib.sha256(self.last_a11y.encode()).hexdigest() node_count = self.last_a11y.count("<") @@ -196,7 +220,7 @@ def _observation(self, task: dict, step_idx: int, budget: Budget, browser_snapshot = None if self.last_browser_snapshot is not None: serialized = json.dumps(self.last_browser_snapshot, ensure_ascii=False) - browser_path = stage / f"step_{step_idx:02d}.dom.json" + browser_path = stage / f"{frame_name}.dom.json" browser_path.write_text(serialized, encoding="utf-8") browser_snapshot = BrowserSnapshot( present=True, @@ -220,15 +244,36 @@ def _observation(self, task: dict, step_idx: int, budget: Budget, ) def _capture(self) -> None: - obs = self.env._get_obs() + for attempt in range(3): + obs = self.env._get_obs() + if obs.get("screenshot"): + break + if attempt < 2: + time.sleep(1) + else: + raise RuntimeError("Environment returned no screenshot after three read-only observations") self.last_screenshot = obs["screenshot"] self.last_a11y = obs.get("accessibility_tree") or "" + result = self.env.controller.execute_python_command(_FOREGROUND_QUERY) + if result is None or result.get("returncode", 0): + raise RuntimeError("Read-only foreground-window observation failed") + info = json.loads(result.get("output") or "{}") + if self.last_a11y: + tree = ET.fromstring(self.last_a11y) + tree.set("foreground_title", info.get("foreground_title", "")) + tree.set("foreground_class", json.dumps(info.get("foreground_class", []))) + self.last_a11y = ET.tostring(tree, encoding="unicode") self.last_browser_snapshot, diagnostic = capture_browser_snapshot(self.env.controller) if not self._cdp_reported: self._cdp_reported = True public = {key: value for key, value in diagnostic.items() if key != "url"} print(json.dumps({"event": "cdp_snapshot", **public}, ensure_ascii=False), flush=True) + def observe(self, task, *, budget, previous): + """Refresh screenshot, A11y and CDP without executing or replaying any action.""" + self._capture() + return self._observation(task, previous.step_idx, budget, previous.last_receipt) + # ------------------------------------------------------------------ 协议 def _seed_setup_cache(self, task_id: str) -> None: """把共享盘已缓存的 setup 文件播种进本次 evaluation_cache(旧 runtime.py:532-539), @@ -291,9 +336,17 @@ def acknowledged(command): stdout = result.get("output", "") stderr = result.get("error", "") returncode = int(result.get("returncode", 0)) + output_ref = None + if kind == "cli": + output = self.run_root / "cli_outputs" / f"action_{self._step_seq:04d}.json" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps({"stdout": stdout, "stderr": stderr, + "returncode": returncode}), encoding="utf-8") + output_ref = str(output) return Receipt( returncode=returncode, stdout=stdout[-RECEIPT_TAIL:], stderr=stderr[-RECEIPT_TAIL:], kind=kind if kind in {"cli", "gui", "wait"} else "gui", acknowledged=True, + full_output_ref=output_ref, ) def execute(self, action: Action, *, budget: Budget, task: dict) -> tuple[Receipt, Observation]: diff --git a/src/tandem_eval/recorder.py b/src/tandem_eval/recorder.py index fe26c6c..2df4145 100644 --- a/src/tandem_eval/recorder.py +++ b/src/tandem_eval/recorder.py @@ -57,6 +57,10 @@ def __init__(self, episode_dir: Path, task: dict, *, a11y_full: bool = False): self.dir.mkdir(parents=True, exist_ok=True) self.task = task self.a11y_full = a11y_full + self.executed_steps = 0 + self.environment_seconds = {} + self.model_usage = {} + self.fast_model = {"choice_cycles": 0, "seconds": 0.0, "cycles_with_image": 0} self.started = time.perf_counter() self._traj = (self.dir / "traj.jsonl").open("w", encoding="utf-8") self._events = (self.dir / "events.jsonl").open("w", encoding="utf-8") @@ -65,6 +69,7 @@ def __init__(self, episode_dir: Path, task: dict, *, a11y_full: bool = False): # ------------------------------------------------------------------ 逐步记录 def traj(self, observation: Observation, action: Action, receipt: Receipt | None) -> None: + self.executed_steps += 1 browser_snapshot = observation.browser_snapshot browser_record = browser_snapshot.to_dict() if browser_snapshot else None if browser_snapshot and browser_snapshot.present and browser_snapshot.path: @@ -89,9 +94,40 @@ def traj(self, observation: Observation, action: Action, receipt: Receipt | None self._traj.flush() def event(self, event: dict) -> None: + if event.get("route") == "environment_timing": + phase = event["phase"] + self.environment_seconds[phase] = self.environment_seconds.get(phase, 0.0) + event["seconds"] + if event.get("route") == "gui" and event.get("ranking"): + ranking = event["ranking"] + self.fast_model["choice_cycles"] += 1 + self.fast_model["seconds"] += ranking["latency_ms"] / 1000 + self.fast_model["cycles_with_image"] += int(ranking["used_image"]) + if event.get("route") == "stage_verified": + event = self._persist_evidence(event) self._events.write(json.dumps(event, ensure_ascii=False) + "\n") self._events.flush() + def task_memory(self, state: dict) -> None: + """Persist the latest host memory and retain its transient evidence before cleanup.""" + _write_json(self.dir / "task_memory.json", self._persist_evidence(state)) + + def _persist_evidence(self, state): + def persist(value): + if isinstance(value, dict): + return {k: persist(v) for k, v in value.items()} + if isinstance(value, list): + return [persist(v) for v in value] + if (isinstance(value, str) and any(part in value for part in ("/obs_stage/", "/cli_outputs/")) + and Path(value).is_file()): + target = self.dir / "evidence" / Path(value).name + target.parent.mkdir(exist_ok=True) + if not target.exists(): + shutil.copy2(value, target) + return str(target) + return value + + return persist(state) + def record_llm_call(self, *, role: str, request: Any, response: Any, **meta: Any) -> None: """慢模型客户端的 llm_sink;全量请求/响应落盘,密钥脱敏。""" self._llm.write( @@ -110,6 +146,16 @@ def record_llm_call(self, *, role: str, request: Any, response: Any, **meta: Any + "\n" ) self._llm.flush() + stats = self.model_usage.setdefault(role, {"calls": 0, "seconds": 0.0, "prompt_tokens": 0, + "completion_tokens": 0, "reasoning_tokens": 0, "cached_tokens": 0, "calls_missing_usage": 0}) + stats["calls"] += 1 + stats["seconds"] += meta.get("elapsed_seconds", 0.0) + usage = response.get("usage") or {} + stats["calls_missing_usage"] += int(not usage) + for key in ("prompt_tokens", "completion_tokens"): + stats[key] += usage.get(key, 0) + stats["reasoning_tokens"] += (usage.get("completion_tokens_details") or {}).get("reasoning_tokens", 0) + stats["cached_tokens"] += (usage.get("prompt_tokens_details") or {}).get("cached_tokens", 0) def screenshot(self, step_idx: int, data: bytes) -> None: target = self.dir / "screenshots" / f"step_{step_idx:02d}.png" @@ -150,7 +196,7 @@ def finalize( "task_id": self.task["id"], "harness": f"tandem {__version__}", "stop_reason": stop_reason, - "steps": steps, + "steps": self.executed_steps if exception is not None else steps, "status": status, "score": score, "exception_type": type(exception).__name__ if exception else None, @@ -163,6 +209,9 @@ def finalize( self.dir / "timing.json", { "wall_seconds": round(time.perf_counter() - self.started, 3), + "environment_seconds": self.environment_seconds, + "model_usage_by_role": self.model_usage, + "fast_model": self.fast_model, "status": status, }, ) diff --git a/src/tandem_eval/rjob_spec.py b/src/tandem_eval/rjob_spec.py index 99250ba..0e8a1fc 100644 --- a/src/tandem_eval/rjob_spec.py +++ b/src/tandem_eval/rjob_spec.py @@ -113,10 +113,17 @@ def _harness_config_section(config: dict, slow_api: dict, fast_api: dict) -> dic return { "execution_mode": execution.get("mode", "step"), "stage_max_actions": execution.get("stage_max_actions", 24), + "goal_recovery_max_calls": int(execution.get("recovery_max_calls", 2)), + "goal_recovery_max_seconds": float(execution.get("recovery_max_seconds", 300)), "route_policy": route.get("policy", "auto"), "allow_cli": bool(route.get("allow_cli", True)), "cli_timeout_seconds": int(route.get("cli_timeout_seconds", 90)), "slow_max_tokens": int(slow_api.get("max_tokens", 16384)), + "slow_verify_max_tokens": int(execution.get("verify_max_tokens", 16384)), + "slow_verify_reasoning_effort": execution.get("verify_reasoning_effort"), + "stage_verify_max_chars": int(execution.get("verify_max_chars", 48000)), + "slow_image_quality": int(execution.get("image_quality", 85)), + "slow_max_request_bytes": int(execution.get("max_request_bytes", 2 * 1024 * 1024)), "slow_timeout_seconds": float(slow_api.get("timeout_seconds", 120)), "slow_reasoning_effort": slow_api.get("reasoning_effort"), "slow_direct_api": bool(slow_api.get("direct_api", False)), @@ -193,7 +200,8 @@ def build_job_spec(*, config: dict, run_id: str, repo_root: Path, "image_stage": _image_stage_section(config), "harness": { "name": "tandem", - "mode": "stage" if harness_config["execution_mode"] == "stage" else "planner_led", + "mode": (harness_config["execution_mode"] + if harness_config["execution_mode"] in {"stage", "goal"} else "planner_led"), "models": {"slow": slow_name, "fast": fast_name}, }, "harness_config": harness_config, diff --git a/src/tandem_eval/runner.py b/src/tandem_eval/runner.py index 8a9d18c..872fefe 100644 --- a/src/tandem_eval/runner.py +++ b/src/tandem_eval/runner.py @@ -7,7 +7,9 @@ from dataclasses import replace from pathlib import Path +from tandem.models.slow import TaskBudgetExceeded from tandem.protocol.models import Observation +from tandem.runtime.goal_stage import ObservationRequired from tandem.runtime.session import TandemSession from .env import budget_of @@ -20,12 +22,21 @@ def run_episode(env, session: TandemSession, recorder: EpisodeRecorder, task: dict, *, max_steps: int, max_wall_seconds: float, - extra_exports: list[str] | None = None) -> dict: + extra_exports: list[str] | None = None, started_monotonic: float | None = None) -> dict: """单题循环:obs → session.act → env.execute → 回执灌回,终态导出交付物并评分。""" - started = time.perf_counter() + started = time.perf_counter() if started_monotonic is None else started_monotonic steps_used = 0 stop_reason = "max_steps" - observation: Observation = env.reset(task) + + def environment_call(phase, function, *args, **kwargs): + before = time.perf_counter() + try: + return function(*args, **kwargs) + finally: + recorder.event({"route": "environment_timing", "phase": phase, + "seconds": time.perf_counter() - before}) + + observation: Observation = environment_call("setup", env.reset, task) outputs: list[dict] = [] events_start = len(session.events) # 本题事件起点:act 后增量落盘(planner/快模型/验证) @@ -34,6 +45,8 @@ def drain_events() -> None: for event in session.events[events_start:]: recorder.event(event) events_start = len(session.events) + if session.goal is not None: + recorder.task_memory(session.goal.memory.snapshot()) while True: budget = budget_of( @@ -50,19 +63,32 @@ def drain_events() -> None: stop_reason = "wall_time_limit" break observation = replace(observation, budget=budget) - action = session.act(observation) - drain_events() + try: + action = session.act(observation) + except ObservationRequired as refresh: + recorder.event({"route": "observation_refresh", "step": steps_used, + "reason": str(refresh), "action_replayed": False}) + observation = environment_call("observe", env.observe, task, budget=budget, previous=observation) + continue + except TaskBudgetExceeded as exhausted: + stop_reason = "wall_time_limit" + recorder.event({"route": "task_budget_exhausted", "step": steps_used, + "reason": str(exhausted), "next": "official_evaluation"}) + break + finally: + drain_events() recorder.event({"step": steps_used, "kind": "action", "action": action.to_dict()}) if getattr(action, "kind", "") in TERMINAL_KINDS: stop_reason = "done" if action.kind == "finish" else action.reason # 终态仍须送达 VM(DONE/FAIL 触发评分钩子,旧 runtime 同款) - env.execute(action, budget=budget, task=task) + environment_call("execute", env.execute, action, budget=budget, task=task) if action.kind == "finish": - outputs = env.export_outputs( + outputs = environment_call( + "export", env.export_outputs, list(action.outputs) + list(extra_exports or []), recorder.dir ) break - receipt, observation = env.execute(action, budget=budget, task=task) + receipt, observation = environment_call("execute", env.execute, action, budget=budget, task=task) recorder.traj(observation, action, receipt) # 环境提供素材属性时写入 episode 目录(真实环境 M3 同样暴露 last_* 属性) shot = getattr(env, "last_screenshot", None) @@ -72,7 +98,9 @@ def drain_events() -> None: if a11y_text: recorder.a11y(steps_used, a11y_text) steps_used += 1 - raw = env.score(task) + raw = environment_call("evaluate", env.score, task) + if session.goal is not None: + recorder.task_memory(session.goal.memory.snapshot()) score = official_score(raw) if raw is not None else None return recorder.finalize( stop_reason=stop_reason, @@ -101,6 +129,7 @@ def run(spec: dict, *, env, session: TandemSession, tasks: list[TaskRef], recorder: EpisodeRecorder | None = None error: BaseException | None = None task: dict = {} + started = time.perf_counter() try: task = env.fetch_task(ref.task_id, ref.domain) recorder = EpisodeRecorder( @@ -108,11 +137,13 @@ def run(spec: dict, *, env, session: TandemSession, tasks: list[TaskRef], ) if llm_tap is not None: llm_tap.current = recorder.record_llm_call + if session.components.set_deadline is not None: + session.components.set_deadline(started + max_wall) session.begin(task["instruction"], spec.get("requirements")) episodes.append( run_episode(env, session, recorder, task, max_steps=max_steps, max_wall_seconds=max_wall, - extra_exports=list(spec.get("export_paths", []))) + extra_exports=list(spec.get("export_paths", [])), started_monotonic=started) ) except BaseException as exc: # 单题失败不终止整个 run error = exc @@ -129,8 +160,13 @@ def run(spec: dict, *, env, session: TandemSession, tasks: list[TaskRef], ) ) finally: + if recorder is not None and session.goal is not None: + recorder.task_memory(session.goal.memory.snapshot()) if llm_tap is not None: llm_tap.current = None + if session.components.set_deadline is not None: + session.components.set_deadline(None) + write_summary(run_root, episodes, spec) # obs_stage 是 episode 内工作区(截图正本已由 recorder 写入 episode 目录, # a11y 摘要已入 traj);episode 结束即清理,避免双份存储 staging = run_root / "obs_stage" / str(ref.task_id) diff --git a/tests/eval/test_browser.py b/tests/eval/test_browser.py index 2bb9219..2a4961e 100644 --- a/tests/eval/test_browser.py +++ b/tests/eval/test_browser.py @@ -33,6 +33,49 @@ def test_cdp_failure_returns_a11y_fallback_diagnostic(): assert diagnostic["available"] is False and "connection refused" in diagnostic["error"] +def test_readonly_javascript_collects_clickable_cards_but_not_their_children_or_occluded_controls(): + import shutil + import subprocess + + import pytest + + node = shutil.which("node") + if node is None: + pytest.skip("Optional Node runtime is unavailable") + namespace = {"__name__": "cdp_test"} + exec(VM_SCRIPT, namespace) + setup = r''' + const body = {tabIndex:-1, onclick:null, cursor:'auto', closest:()=>null, + matches:s=>s.includes('body')}; + const make = (text, x, parent=body, native=false) => ({ + tagName:native?'BUTTON':'DIV', type:'', tabIndex:-1, onclick:native?null:()=>{}, + cursor:native?'auto':'pointer', parentElement:parent, childNodes:[{nodeType:3,textContent:text}], + labels:[], innerText:text, disabled:false, checkVisibility:()=>true, + getAttribute:()=>null, matches:s=>native&&s.includes('button'), closest:()=>null, + querySelector:()=>null, querySelectorAll:()=>[], + getBoundingClientRect:()=>({left:x,top:20,right:x+100,bottom:60,width:100,height:40}), + contains(other){return other===this||other.parentElement===this;}, + cloneNode(){return {innerText:text,querySelectorAll:()=>[]};} + }); + const card=make('Afternoon class',20); + const child=make('Child label',20,card); + const hidden=make('Covered button',200,body,true); + const cover=make('Overlay',200); + global.getComputedStyle=e=>({cursor:e.cursor,display:'block',visibility:'visible',opacity:'1'}); + global.innerWidth=800;global.innerHeight=600; + global.window={screenX:0,screenY:0,outerWidth:800,innerWidth:800,outerHeight:600,innerHeight:600}; + global.location={href:'https://example.test/'};global.devicePixelRatio=1; + global.document={title:'Synthetic cards',visibilityState:'visible',activeElement:null, + getElementById:()=>null, + querySelectorAll:s=>s.startsWith('a[href]')?[card,child,hidden]:[], + elementFromPoint:x=>x<150?child:cover}; + ''' + run = subprocess.run([node, "-e", setup + "\nconsole.log(JSON.stringify(" + namespace["EXPRESSION"] + + "));"], capture_output=True, text=True, check=False) + assert run.returncode == 0, run.stderr + assert [e["name"] for e in json.loads(run.stdout)["elements"]] == ["Afternoon class"] + + def test_vm_websocket_frame_parser_executes_with_stdlib_only(): namespace = {"__name__": "cdp_test"} exec(VM_SCRIPT, namespace) diff --git a/tests/eval/test_config.py b/tests/eval/test_config.py index 5739758..7b5f0fd 100644 --- a/tests/eval/test_config.py +++ b/tests/eval/test_config.py @@ -69,8 +69,9 @@ def test_models_are_named_api_registries(): for api in slow["apis"].values(): assert set(api) >= {"base_url", "api_key", "model"} for api in fast["apis"].values(): - assert api["backend"] in {"jev", "kev"} - assert api.get("endpoint") or api.get("endpoint_file") + assert api["backend"] in {"jev", "kev", "planner"} + if api["backend"] != "planner": + assert api.get("endpoint") or api.get("endpoint_file") def test_paths_are_absolute_and_credentials_inlined(): diff --git a/tests/eval/test_experiment_script.py b/tests/eval/test_experiment_script.py index 9c2d3ea..9773b32 100644 --- a/tests/eval/test_experiment_script.py +++ b/tests/eval/test_experiment_script.py @@ -54,6 +54,16 @@ def test_invalid_experiment_name_is_rejected(tmp_path): assert not (tmp_path / "bad name").exists() +def test_dry_run_selects_goal_mode_without_changing_script_default(tmp_path): + result = subprocess.run([ + "bash", str(SCRIPT), "--experiment", "goal-smoke", "--tasks", "008", "--execution-mode", "goal", + "--stage-max-actions", "2", "--dry-run"], capture_output=True, text=True, + env={**os.environ, "TANDEM_EXPERIMENT_ROOT": str(tmp_path)}) + assert result.returncode == 0, result.stderr + manifest = json.loads((tmp_path / "goal-smoke/manifest.json").read_text()) + assert manifest["execution_mode"] == "goal" and manifest["stage_max_actions"] == 2 + + def test_dry_run_uses_example_when_private_config_is_missing(tmp_path): env = { **os.environ, diff --git a/tests/eval/test_mock_chain.py b/tests/eval/test_mock_chain.py index 5391883..a0a4933 100644 --- a/tests/eval/test_mock_chain.py +++ b/tests/eval/test_mock_chain.py @@ -41,9 +41,10 @@ def _spec(tmp_path: Path, **overrides) -> dict: return base -@pytest.mark.parametrize("mode", ["step", "stage"]) +@pytest.mark.parametrize("mode", ["step", "stage", "goal"]) def test_full_mock_chain_produces_renderable_result_dir(tmp_path, mode): - episodes = run_from_spec(_spec(tmp_path, harness_config={"execution_mode": mode})) + episodes = run_from_spec(_spec(tmp_path, harness_config={"execution_mode": mode}, + requirements=None if mode == "goal" else [])) assert episodes and episodes[0]["status"] == "scored" run_root = tmp_path / "run" episode_dir = run_root / "episodes" / "mock-mock-000" @@ -64,6 +65,12 @@ def test_full_mock_chain_produces_renderable_result_dir(tmp_path, mode): assert episode["stop_reason"] == "done" and episode["score"] == 1.0 summary = json.loads((run_root / "summary.json").read_text()) assert summary["scored"] == 1 and summary["average_score"] == 1.0 + if mode == "goal": + memory = json.loads((episode_dir / "task_memory.json").read_text()) + assert memory["work_items"][0]["status"] == "done" + events = [json.loads(line) for line in (episode_dir / "events.jsonl").read_text().splitlines()] + assert any(e.get("route") == "stage_verified" for e in events) + assert any(e.get("route") == "observation_refresh" for e in events) # 渲染(demo 占位)可离线打开 page = render_summary.render(run_root) @@ -95,6 +102,51 @@ def fetch_task(self, task_id: str, domain: str = "") -> dict: assert summary["failed_unscored"] == 1 and summary["average_score"] == 1.0 +def test_agent_budget_exhausted_inside_model_still_reaches_official_scoring(tmp_path): + from tandem.config import SessionConfig + from tandem.models.slow import TaskBudgetExceeded + from tandem.runtime.session import TandemSession + + def planner(*args, **kwargs): + raise TaskBudgetExceeded("Agent task budget exhausted") + + components = make_components(planner, lambda *a, **kw: None, FixedSelector()) + root = tmp_path / "run" + episodes = run(_spec(tmp_path), env=MockEnv(base_dir=root / "mock_env"), + session=TandemSession(components, config=SessionConfig()), + tasks=[TaskRef("one", "mock")], run_root=root) + assert episodes[0]["stop_reason"] == "wall_time_limit" + assert episodes[0]["status"] == "scored" + assert episodes[0]["score"] is not None + + +def test_requirement_timeout_has_a_deadline_and_flushes_summary_before_next_task(tmp_path): + from tandem.config import SessionConfig + from tandem.runtime.session import TandemSession + + deadlines = [] + root = tmp_path / "run" + + class Env(MockEnv): + def fetch_task(self, task_id, domain=""): + if task_id == "second": + assert json.loads((root / "summary.json").read_text())["failed_unscored"] == 1 + return super().fetch_task(task_id, domain) + + def requirements(instruction): + assert isinstance(deadlines[-1], float) + raise TimeoutError("Task deadline exhausted during requirements") + + components = make_components(lambda *a, **kw: None, lambda *a, **kw: None, FixedSelector()) + components.requirements = requirements + components.set_deadline = deadlines.append + episodes = run(_spec(tmp_path, requirements=None), env=Env(base_dir=root / "mock_env"), + session=TandemSession(components, config=SessionConfig()), + tasks=[TaskRef("first"), TaskRef("second")], run_root=root) + assert [item["status"] for item in episodes] == ["failed_unscored", "failed_unscored"] + assert deadlines[1] is None and deadlines[3] is None + + def test_entry_cli_from_spec_file(tmp_path, capsys): spec_path = tmp_path / "spec.json" spec_path.write_text(json.dumps(_spec(tmp_path), ensure_ascii=False), encoding="utf-8") diff --git a/tests/eval/test_osworld_env.py b/tests/eval/test_osworld_env.py index 10fe121..ed7bdec 100644 --- a/tests/eval/test_osworld_env.py +++ b/tests/eval/test_osworld_env.py @@ -4,6 +4,7 @@ import json import struct +import xml.etree.ElementTree as ET from pathlib import Path from types import ModuleType @@ -42,10 +43,10 @@ def test_gui_code_matches_legacy_pyautogui_semantics(): assert gui_code(GuiAction("type", { "text": "42", "input_mode": "replace", "click": [650, 102], "press_f2": True, })) == ("pyautogui.click(x=650, y=102); pyautogui.press('f2'); " - "pyautogui.hotkey('ctrl', 'a'); pyautogui.write('42')") + "pyautogui.hotkey('ctrl', 'a'); pyautogui.write('42', interval=0.03)") assert gui_code(GuiAction("type", { "text": "cmd", "input_mode": "insert", "click": None, "press_f2": False, - })) == "pyautogui.write('cmd')" + })) == "pyautogui.write('cmd', interval=0.03)" def test_action_code_cli_wait_and_terminals(): @@ -62,6 +63,64 @@ def test_png_size_parses_ihdr(): assert png_size(_png(1920, 1080)) == (1920, 1080) +def test_read_only_refresh_keeps_cdp_receipt_and_prior_evidence(tmp_path, env, monkeypatch): + snapshot = {"elements": [{"index": 1, "role": "entry", "name": "Receipt Date"}]} + monkeypatch.setattr("tandem_eval.osworld.capture_browser_snapshot", + lambda controller: (snapshot, {"status": "ok"})) + task = _task_file(tmp_path) + first = env.reset(task) + budget = Budget(steps_remaining=10, wall_seconds_remaining=60) + receipt, previous = env.execute(GuiAction("click", {"x": 1, "y": 2}), budget=budget, task=task) + steps, commands = list(env.env.steps), list(env.env.controller.commands) + before = Path(previous.screenshot.path).read_bytes() + refreshed = env.observe(task, budget=budget, previous=previous) + assert refreshed.protocol_version == "0.2" and refreshed.last_receipt is receipt + assert refreshed.step_idx == previous.step_idx + assert env.env.steps == steps + assert env.env.controller.commands[:-1] == commands + assert "_NET_ACTIVE_WINDOW" in env.env.controller.commands[-1] # Read-only window observation. + tree = ET.fromstring(Path(refreshed.accessibility_tree.path).read_text()) + assert tree.get("foreground_title") == "main-window" + assert json.loads(tree.get("foreground_class")) == ["app", "App"] + assert refreshed.browser_snapshot.source == "cdp" + assert json.loads(Path(refreshed.browser_snapshot.path).read_text()) == snapshot + assert refreshed.screenshot.path != previous.screenshot.path != first.screenshot.path + assert Path(previous.screenshot.path).read_bytes() == before + + +def test_empty_screenshot_retries_only_observation(tmp_path, env, monkeypatch): + from unittest.mock import Mock + + task = _task_file(tmp_path) + previous = env.reset(task) + valid = env.env._get_obs() + observe = Mock(side_effect=[{"screenshot": None}, {"screenshot": b""}, valid]) + monkeypatch.setattr(env.env, "_get_obs", observe) + monkeypatch.setattr("tandem_eval.osworld.time.sleep", lambda seconds: None) + budget = Budget(steps_remaining=5, wall_seconds_remaining=30) + refreshed = env.observe(task, budget=budget, previous=previous) + assert observe.call_count == 3 and env.env.steps == [] + assert refreshed.step_idx == previous.step_idx and refreshed.last_receipt is None + assert Path(refreshed.screenshot.path).read_bytes() == valid["screenshot"] + + +def test_repeated_empty_screenshot_is_explicit_observation_failure(tmp_path, env, monkeypatch): + from unittest.mock import Mock + + task = _task_file(tmp_path) + previous = env.reset(task) + observe = Mock(return_value={"screenshot": None}) + monkeypatch.setattr(env.env, "_get_obs", observe) + monkeypatch.setattr("tandem_eval.osworld.time.sleep", lambda seconds: None) + with pytest.raises(RuntimeError, match="no screenshot after three read-only"): + env.observe(task, budget=previous.budget, previous=previous) + assert observe.call_count == 3 and env.env.steps == [] + + +def test_right_click_codegen_uses_real_mouse_button(): + assert gui_code(GuiAction("right_click", {"x": 20, "y": 30})) == "pyautogui.rightClick(x=20, y=30)" + + # ------------------------------------------------------------------ adapter class FakeController: def __init__(self): @@ -71,6 +130,9 @@ def __init__(self): def execute_python_command(self, command): self.commands.append(command) + if "_NET_ACTIVE_WINDOW" in command: + return {"returncode": 0, "output": json.dumps({ + "foreground_title": "main-window", "foreground_class": ["app", "App"]})} return {"returncode": 0, "output": json.dumps( {"returncode": 0, "stdout": "ok-out", "stderr": ""})} diff --git a/tests/eval/test_rjob_client.py b/tests/eval/test_rjob_client.py index 53dd21d..734284f 100644 --- a/tests/eval/test_rjob_client.py +++ b/tests/eval/test_rjob_client.py @@ -146,6 +146,18 @@ def test_cli_budget_override_merges_with_config(tmp_path): assert spec["budget"] == {"max_steps": 3, "max_wall_seconds": 60.0} +def test_goal_recovery_budget_is_taken_from_execution_config(tmp_path): + config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy")) + config["harness"]["execution"] = { + "mode": "goal", "recovery_max_calls": 3, "recovery_max_seconds": 240, + } + spec = build_job_spec(config=config, run_id="goal", repo_root=tmp_path, + spec_overrides={"benchmark": "mock"}) + assert spec["harness_config"]["execution_mode"] == "goal" + assert spec["harness_config"]["goal_recovery_max_calls"] == 3 + assert spec["harness_config"]["goal_recovery_max_seconds"] == 240 + + def test_watch_until_terminal_and_timeout(): runner = FakeRunner({"get": ["running", "succeeded"]}) client = RjobClient(runner) diff --git a/tests/harness/conftest.py b/tests/harness/conftest.py index f7317ab..e91bc14 100644 --- a/tests/harness/conftest.py +++ b/tests/harness/conftest.py @@ -7,6 +7,7 @@ from pathlib import Path import pytest +from PIL import Image from tandem.components import Components from tandem.gui.candidates import ActionCandidateBuilder @@ -61,7 +62,7 @@ def make_observation(tmp_path: Path, step_idx: int, *, receipt: Receipt | None = tree_xml: str | None = A11Y_XML, steps: int = 50, wall: float = 1800.0, screenshot_sha: str = SHA) -> Observation: shot = tmp_path / f"step_{step_idx:02d}.png" - shot.write_bytes(b"png-bytes") + Image.new("RGB", (64, 64), "white").save(shot) a11y_path = None if tree_xml is not None: a11y_path = str(tmp_path / f"a11y_{step_idx:02d}.xml") diff --git a/tests/harness/test_execution_control.py b/tests/harness/test_execution_control.py new file mode 100644 index 0000000..b75af0b --- /dev/null +++ b/tests/harness/test_execution_control.py @@ -0,0 +1,184 @@ +"""Synthetic, task-independent execution-control regressions; not benchmark results.""" + +from dataclasses import replace + +from tandem.gui.candidates import ActionCandidate +from tandem.gui.compile import gui_action +from tandem.gui.goal_candidates import goal_candidates +from tandem.gui.interaction import GuiInteractionController +from tandem.memory.context import execution_context, planning_context +from tandem.memory.state import TaskMemory +from tandem.planning.goal_planner import goal_error + +from .conftest import build_space +from .test_goal_execution import _space +from .test_goal_progress import FORM +from .test_goal_stage import plan, requirements, result + + +def _setup(inputs=()): + memory = TaskMemory("Edit and save", requirements()) + memory.start({**plan(), "inputs": list(inputs)}, 0) + targets, _ = _space(FORM) + controller = GuiInteractionController() + space = build_space(targets) + controller.finalize(space, targets, memory.active_stage, memory, FORM) + return controller, memory, targets + + +def _pending(controller, operation="keypress", sequence=None): + return {"operation": operation, "action": {"parameters": {"sequence": sequence}}, + "control_scope": controller.control["scope_key"], "control_state": controller.content_state} + + +def test_pixels_and_stage_rewording_do_not_release_a_failed_execution_path(): + controller, memory, targets = _setup() + for _ in range(4): + controller.receive(_pending(controller, sequence=[["tab"]]), "unknown", True, 0, targets=targets) + assert controller.checkpoint_due + assert controller.local_effect == "unknown" + update = result(unit_met=False, delivery_state="none", stage_outcome="not_met") + controller.finish(memory, update, [{"evidence_ref": "A2"}]) + memory.apply(update, {"A2": {}}) + assert memory.work_items["R1:unit:1"]["recovery_required"] + assert "execution recovery" in goal_error(plan(), memory, True) + revised = {**plan(), "method": "Different words", "recovery_strategy": "alternate_interaction"} + assert not goal_error(revised, memory, True) + memory.start(revised, 4) + space = build_space(targets) + space.target_heads["keypress"] += [ActionCandidate( + "tab", "keypress", "Tab", metadata={"key_sequence": [["tab"]]})] + controller.finalize(space, targets, memory.active_stage, memory, FORM) + assert not any(c.id in {"tab", "KEYPRESS_TAB"} for c in space.candidates()) + assert any(c.operation == "click" for c in space.candidates()) + assert planning_context(memory)["execution_constraints"][0]["blocked_families"] == ["focus_traversal"] + + +def test_new_known_focus_positions_are_allowed_until_the_traversal_cycles(): + controller, memory, targets = _setup() + base = next(t for t in targets if t.name == "Title *") + fields = [replace(base, name=f"Field {i}", semantic_key=f"field-{i}", focused=False) + for i in range(8)] + controller.finalize(build_space(fields), fields, memory.active_stage, memory, FORM) + for index in range(8): + observed = [replace(t, focused=i == index) for i, t in enumerate(fields)] + controller.receive(_pending(controller, sequence=[["tab"]]), "unknown", True, 0, targets=observed) + assert not controller.checkpoint_due + observed = [replace(t, focused=i == 0) for i, t in enumerate(fields)] + controller.receive(_pending(controller, sequence=[["tab"]]), "unknown", True, 0, targets=observed) + assert controller.checkpoint_due + + +def test_new_observed_content_allows_continuous_navigation_without_semantic_verify(): + controller, memory, targets = _setup() + base = next(t for t in targets if t.name == "Title *") + for index in range(12): + observed = [replace(base, name=f"Visible content section {index}")] + controller.receive(_pending(controller, operation="scroll"), "unknown", True, 0, targets=observed) + controller.finalize(build_space(observed), observed, memory.active_stage, memory, FORM) + assert controller.local_effect == "controls_changed" + assert not controller.checkpoint_due and not controller.control["blocked_families"] + + +def test_confirmed_field_effect_is_progress_but_does_not_erase_another_failed_path(): + controller, _, targets = _setup() + controller.control.update(blocked_families=["scroll"], recovery_required=True) + pending = _pending(controller, operation="type_text") + controller.receive(pending, "unknown", True, 0, targets=targets) + assert controller.control["blocked_families"] == ["scroll"] + controller.receive(pending, "met", True, 0, targets=targets) + assert controller.local_effect == "confirmed_input" + assert controller.control["without_effect"] == 0 + assert controller.control["blocked_families"] == ["scroll"] + assert controller.control["recovery_required"] + + +def test_final_scope_rejects_a_commit_added_for_a_background_field(): + value = {"label": "Command", "target_hint": "Title", "text": "Example(1)", + "input_mode": "replace", "kind": "command", "commit": "enter"} + controller, memory, targets = _setup([value]) + # Establish a real command-input dispatch before the new foreground modal appears. + space = build_space(targets) + space.target_heads["type_text"] = [ActionCandidate( + "command", "type_text", "Type command", target_id=next(t.id for t in targets if t.name == "Title *"), + metadata={"text": value["text"], "target_hint": "Title", "input_mode": "replace"})] + controller.constrain(space, targets, memory.active_stage, memory) + typed = space.target_heads["type_text"][0] + controller.receive({"operation": "type_text", + "interaction_binding": typed.metadata["interaction_binding"]}, + "unknown", True, 0) + modal = ('' + '') + memory.active_stage.update(target_document="expected-document.pptx", document_recovery=True) + space = goal_candidates(build_space(targets), targets, modal, []) + space.target_heads["visual"] = [ActionCandidate( + "close", "click", "Close the visible confirmation", source="visual_grounder", + metadata={"visual_point": [650, 420], "blocking_overlay": True, + "overlay_name": "Confirmation", "overlay_bounds": [400, 400, 300, 200]})] + context = controller.finalize(space, targets, memory.active_stage, memory, modal) + assert not any(c.metadata.get("commit_action") for c in space.candidates()) + assert "Confirmation" in context["next_intent"] + assert any(c.id == "close" for c in space.candidates()) + assert gui_action(next(c for c in space.candidates() if c.id == "close"), None).action_type == "click" + + +def test_non_delivery_source_facts_do_not_clear_an_active_modal_failure(): + controller, memory, targets = _setup() + controller.control.update(global_scope=True, attempts={"focus_traversal": 5}) + update = result(stage_outcome="met", unit_met=False, delivery_state="none", progress_kind="source", + facts=[{"key": "source.note", "value": "An unrelated observed note", + "status": "confirmed", + "scope": "task", "source_ref": "source.txt", "evidence_refs": ["A2"]}]) + controller.finish(memory, update, [{"evidence_ref": "A2"}]) + memory.apply(update, {"A2": {}}) + assert controller.control["blocked_families"] == ["focus_traversal"] + assert controller.control["recovery_required"] + + +def test_a_physical_modal_failure_survives_switching_work_units(): + req = requirements()[0] + req.update(units=["First section", "Second section"], unit_fact_keys=[[], []]) + memory = TaskMemory("Edit two sections", [req]) + controller = GuiInteractionController() + modal = '' + + def space(): + value = goal_candidates(build_space([]), [], modal, []) + value.target_heads["visual"] = [ActionCandidate( + "visible-button", "click", "Visible button", source="visual_grounder", + metadata={"visual_point": [30, 40], "blocking_overlay": True, + "overlay_name": "Confirmation", "overlay_bounds": [10, 20, 100, 100]})] + return value + + memory.start(plan(), 0) + controller.finalize(space(), [], memory.active_stage, memory, modal) + for _ in range(4): + controller.receive(_pending(controller, sequence=[["tab"]]), "unknown", True, 0) + update = result(stage_outcome="not_met", unit_met=False, delivery_state="none") + controller.finish(memory, update, [{"evidence_ref": "A2"}]) + memory.apply(update, {"A2": {}}) + memory.start({**plan(), "work_item_id": "R1:unit:2"}, 4) + choices = space() + controller.finalize(choices, [], memory.active_stage, memory, modal) + assert controller.control["global_scope"] + assert not any(c.id == "KEYPRESS_TAB" for c in choices.candidates()) + assert execution_context(memory)["execution_constraints"][0]["blocked_families"] == ["focus_traversal"] + + +def test_confirmed_required_source_content_can_release_nonmodal_browsing(): + req = requirements()[0] + req.update(facts_needed=[{"key": "source.required", "description": "Required source section"}], + unit_fact_keys=[["source.required"]]) + memory = TaskMemory("Read required source", [req]) + memory.start({**plan(), "purpose": "prepare"}, 0) + controller = GuiInteractionController() + targets, _ = _space(FORM) + controller.finalize(build_space(targets), targets, memory.active_stage, memory, FORM) + controller.control.update(blocked_families=["scroll"], attempts={"scroll": 4}, recovery_required=True) + update = result(unit_met=False, delivery_state="none", progress_kind="source", facts=[{ + "key": "source.required", "value": "Confirmed requested section", "status": "confirmed", + "scope": "task", "source_ref": "source.txt", "evidence_refs": ["A2"]}]) + controller.finish(memory, update, [{"evidence_ref": "A2"}]) + assert not controller.control["blocked_families"] + assert not controller.control["recovery_required"] diff --git a/tests/harness/test_focus.py b/tests/harness/test_focus.py index aa5744c..b312dfe 100644 --- a/tests/harness/test_focus.py +++ b/tests/harness/test_focus.py @@ -14,6 +14,14 @@ def test_menu_extraction_with_alt_mnemonic(): assert menus[0]["key_sequence"] == [["esc"], ["alt", "f"], ["i"]] +def test_menu_item_mnemonic_does_not_confirm_the_dialog_it_opens(): + tree = ''' + + ''' + _, menus = input_and_menu_context(tree) + assert menus[0]["key_sequence"] == [["esc"], ["alt", "f"], ["o"]] + + def test_inputs_include_focused_editable_with_bounds(): inputs, _ = input_and_menu_context(A11Y_XML) focused = [item for item in inputs if item["focused"]] diff --git a/tests/harness/test_goal_blocker.py b/tests/harness/test_goal_blocker.py new file mode 100644 index 0000000..ede21d0 --- /dev/null +++ b/tests/harness/test_goal_blocker.py @@ -0,0 +1,117 @@ +"""Generic binding/blocker regressions derived from recorded failures; synthetic fixtures.""" + +from dataclasses import replace + +from tandem.gui.candidates import ActionCandidate +from tandem.gui.goal_candidates import _input_target, goal_candidates +from tandem.gui.interaction import GuiInteractionController +from tandem.memory.context import execution_context, planning_context +from tandem.memory.state import TaskMemory + +from .conftest import build_space +from .test_goal_execution import _space +from .test_goal_progress import FORM +from .test_goal_stage import plan, requirements, result + + +def test_whole_form_button_cannot_steal_native_choice_fields(): + targets, _ = _space(FORM) + base = targets[0] + fields = [replace(base, id="country", role="combo_box", name="Country of Birth", + input_type="select-one", operations=("click",)), + replace(base, id="level", role="combo_box", name="Education Level", + input_type="select-one", operations=("click",)), + replace(base, id="form", role="push_button", operations=("click",), input_type="", + name="Questionnaire Country of Birth China Education Level Bachelor's Save Submit")] + for label, text, expected in [("Country of Birth", "China", "country"), + ("Education Level", "Bachelor's", "level")]: + value = {"label": label, "target_hint": label, "text": text, + "kind": "choice", "input_mode": "replace"} + assert _input_target(value, fields).id == expected + + +def setup_controller(): + req = requirements()[0] + req.update(units=["First unit", "Second unit"], unit_fact_keys=[[], []]) + memory = TaskMemory("Complete two units", [req]) + memory.start(plan(), 0) + controller = GuiInteractionController() + targets, _ = _space(FORM) + return controller, memory, targets + + +def test_rejected_zero_action_completion_survives_switching_work_units(): + controller, memory, targets = setup_controller() + space = goal_candidates(build_space(targets), targets, FORM, []) + controller.finalize(space, targets, memory.active_stage, memory, FORM, "S1:O0") + assert "done" in space.controls + rejected = result(stage_outcome="not_met", unit_met=False, delivery_state="none", + summary="The window was not maximized", evidence_refs=["S1:O0"]) + controller.finish(memory, rejected, []) + memory.apply(rejected, {"S1:O0": {}}) + memory.start({**plan(), "work_item_id": "R1:unit:2"}, 0) + space = goal_candidates(build_space(targets), targets, FORM, []) + controller.finalize(space, targets, memory.active_stage, memory, FORM, "S2:O0") + assert "done" not in space.controls + assert memory.active_blocker["kind"] == "rejected_completion" + assert execution_context(memory)["active_blocker"] == planning_context(memory)["active_blocker"] + assert controller.control["failed_stages"] == 1 + + +def test_new_related_control_value_releases_rejected_completion_with_evidence(): + controller, memory, targets = setup_controller() + controller.finalize(goal_candidates(build_space(targets), targets, FORM, []), + targets, memory.active_stage, memory, FORM, "S1:O0") + controller.finish(memory, result(stage_outcome="not_met", unit_met=False, delivery_state="none"), []) + observed = [replace(t, value="New observed field value") for t in targets] + space = goal_candidates(build_space(observed), observed, FORM, []) + controller.finalize(space, observed, memory.active_stage, memory, FORM, "S1:O1") + assert "done" in space.controls + assert memory.active_blocker["status"] == "resolved" + assert memory.active_blocker["resolution_evidence_refs"] == ["S1:O1"] + + +def test_short_goal_is_not_replaced_by_acceptance_criterion(): + controller, memory, targets = setup_controller() + memory.active_stage.update(goal="Maximize the current window", completion_criteria="Window is maximized") + context = controller.finalize(goal_candidates(build_space(targets), targets, FORM, []), + targets, memory.active_stage, memory, FORM) + view = controller.choice_context({**execution_context(memory), **context}) + assert view["objective"] == "Maximize the current window" + assert view["exit_condition"] == "Window is maximized" + + +def test_unrelated_modal_names_share_one_owner_and_suspend_future_goal(): + controller, memory, _ = setup_controller() + tree = '' + controls = [] + for name in ["Default office software prompt", "Default software notice", "Editor prompt"]: + space = goal_candidates(build_space([]), [], tree, []) + space.target_heads["visual"] = [ActionCandidate("close", "click", "Close", source="visual_grounder", + metadata={"visual_point": [30, 40], "blocking_overlay": True, "overlay_name": name, + "overlay_kind": "obstruction", "overlay_bounds": [10, 20, 100, 100]})] + context = controller.finalize(space, [], memory.active_stage, memory, tree, "S1:O0") + controls.append(controller.control) + controller.control["ground_calls"] = 2 + view = controller.choice_context({**execution_context(memory), **context}) + assert "Restore input access" in view["objective"] + assert view["inputs"] == [] and "Form completed" not in view["exit_condition"] + assert controls[0] is controls[1] is controls[2] + assert controls[-1]["ground_calls"] == 2 + controller.finalize(goal_candidates(build_space([]), [], tree, []), [], + memory.active_stage, memory, tree, "S1:O2") + assert memory.active_blocker["status"] == "resolved" + assert memory.active_blocker["resolution_evidence_refs"] == ["S1:O2"] + + +def test_required_task_dialog_is_not_dismissed_after_failure(): + controller, memory, _ = setup_controller() + tree = '' + space = goal_candidates(build_space([]), [], tree, []) + space.target_heads["visual"] = [ActionCandidate("save", "click", "Save", source="visual_grounder", + metadata={"visual_point": [30, 40], "blocking_overlay": True, "overlay_name": "Save", + "overlay_kind": "task_dialog", "overlay_bounds": [10, 20, 100, 100]})] + controller.finalize(space, [], memory.active_stage, memory, tree) + controller.control["recovery_required"] = True + context = controller.finalize(space, [], memory.active_stage, memory, tree) + assert not context["scope_recovery"] diff --git a/tests/harness/test_goal_delivery_loop.py b/tests/harness/test_goal_delivery_loop.py new file mode 100644 index 0000000..0a23800 --- /dev/null +++ b/tests/harness/test_goal_delivery_loop.py @@ -0,0 +1,187 @@ +"""Synthetic regressions for the execution/observation/delivery handoffs; not benchmark scores.""" + +import json +from dataclasses import replace +from unittest.mock import Mock + +import pytest + +from tandem.gui.goal_candidates import ( + foreground_tree, + goal_candidates, + modal_frontier, + portal_needs_grounding, +) +from tandem.gui.interaction import GuiInteractionController, target_effect +from tandem.gui.semantics import enrich_targets +from tandem.memory.context import execution_context, planning_context, verification_context +from tandem.memory.state import TaskMemory +from tandem.runtime.goal_stage import ObservationRequired +from tandem.verification.verifier import verify_observation + +from .conftest import build_space, build_targets, cli_receipt, make_observation +from .test_goal_memory import session +from .test_goal_stage import plan, requirements, result + +PORTAL = ''' + + + +''' + + +def test_coordinate_free_portal_is_retained_for_visual_confirmation(): + tree = foreground_tree(PORTAL) + assert "_Open Open" in tree and "Submit" in tree + assert portal_needs_grounding(tree) == "Open Files" + # Coordinate-free nodes are not turned into invented click coordinates. + assert not [t for t in build_targets(tree) if t.name == "_Open Open"] + + +def test_coordinate_free_portal_requests_grounder_despite_rich_background(tmp_path): + s = session() + s.begin("Upload the document", requirements()) + s.goal.memory.start(plan(), 0) + s.components.grounder = Mock(return_value=[]) + s.act(make_observation(tmp_path, 0, tree_xml=PORTAL)) + s.components.grounder.assert_called_once() + + +def test_native_dialog_keeps_focus_and_selection_keys(): + tree = ' + ''' + assert not enrich_targets(build_targets(tree), tree, (1920, 1080)) + s = session() + s.begin("Open the reference", requirements()) + s.goal.memory.start(plan(), 0) + s.components.grounder = Mock(return_value=[]) + s.act(make_observation(tmp_path, 0, tree_xml=tree)) + s.components.grounder.assert_called_once() + + +def test_field_hint_does_not_replace_preservation_goal(): + controller = GuiInteractionController() + context = {"goal": "Read the reference in a NEW tab and preserve the filled form", + "completion_criteria": "Reference read and form retained", "inputs": [], + "next_intent": "Type the reference URL into the address bar", "constraints": [], + "method": "Open a new tab first", "guidance": "Never replace the existing form tab"} + view = controller.choice_context(context) + assert view["objective"] == context["goal"] + assert view["next_action_hint"] == context["next_intent"] + assert view["guidance"] == context["guidance"] + + +def test_opened_document_consumes_its_now_absent_path_entry(): + memory = TaskMemory("Read reference", requirements()) + value = {"label": "Reference path", "target_hint": "Enter location or URL", "text": "/tmp/reference.png", + "input_mode": "replace", "commit": "enter"} + memory.start({**plan(), "purpose": "prepare", "inputs": [value]}, 0) + controller = GuiInteractionController() + controller.stage_id = memory.active_stage["id"] + key = json.dumps([value["target_hint"].casefold(), value["text"]], ensure_ascii=False) + controller.fields[key] = {"target": value["target_hint"], "expected_value": value["text"], + "phase": "submitted_unconfirmed", "commit": "enter"} + tree = '' + space = goal_candidates(build_space([]), [], tree, [value]) + context = controller.finalize(space, [], memory.active_stage, memory, tree) + assert context["unbound_inputs"] == [] and controller.fields[key]["phase"] == "consumed" + assert controller.choice_context({**execution_context(memory), **context})["inputs"] == [] + + +def test_unrelated_focused_field_cannot_disprove_a_visual_input(): + target = replace(build_targets()[0], name="Address and search bar", value="old", focused=True) + pending = {"operation": "type_text", "field_label": "Enter location or URL", + "expected_text": "/tmp/reference.png", "input_mode": "replace"} + assert target_effect(pending, [target]) == "unknown" + + +def test_unpublished_working_copy_is_not_saved_delivery_then_can_be_published(): + memory = TaskMemory("Update the board deck", requirements()) + memory.start(plan(), 0) + first = result(artifacts=[{"target_path": "/tmp/working.pptx", "working_path": "/tmp/working.pptx", + "save_state": "saved", "evidence_refs": ["A2"]}]) + memory.apply(first, {"A2": {}}) + assert memory.work_items["R1:unit:1"]["status"] == "pending" + assert first["delivery_state"] == "implemented" and not first["unit_met"] + assert planning_context(memory)["pending_publications"] + memory.start({**plan(), "purpose": "save"}, 2) + merely_open = result(evidence_refs=["A3"]) + memory.apply(merely_open, {"A3": {}}) + assert not merely_open["unit_met"] # Opening the temporary copy does not publish it. + memory.start({**plan(), "purpose": "save"}, 3) + final = result(evidence_refs=["A3"], artifacts=[{ + "target_path": "/home/user/Desktop/board.pptx", "working_path": "/home/user/Desktop/board.pptx", + "save_state": "saved", "replaces_working_path": "/tmp/working.pptx", "evidence_refs": ["A3"]}]) + memory.apply(final, {"A3": {}}) + assert memory.work_items["R1:unit:1"]["status"] == "done" + assert not planning_context(memory)["pending_publications"] + + +def test_rollback_revokes_requirement_status_not_confirmed_source_facts(): + reqs = requirements() + reqs[0]["units"].append("Another section") + reqs[0]["unit_fact_keys"].append([]) + memory = TaskMemory("Complete form", reqs) + memory.start(plan(), 0) + fact = {"key": "source", "value": "Stable source", "status": "confirmed", "scope": "task", + "source_ref": "input.pdf", "evidence_refs": ["A2"]} + memory.apply(result(facts=[fact], requirement_checks=[{ + "id": "R1", "status": "met", "evidence_refs": ["A2"]}]), {"A2": {}}) + memory.start({**plan(), "work_item_id": "R1:unit:2", "purpose": "prepare"}, 2) + memory.apply(result(unit_met=False, delivery_state="none", evidence_refs=["A3"], + invalidated_work_item_ids=["R1:unit:1"]), {"A3": {}}) + assert memory.requirements[0]["status"] == "uncertain" + assert memory.work_items["R1:unit:1"]["status"] == "pending" + assert memory.facts["source"]["value"] == "Stable source" + + +def test_real_dispatch_resets_terminal_rejection_counter(tmp_path): + s = session() + s.begin("Edit", requirements()) + s.goal.memory.start({**plan(), "route": "cli", "command": "true"}, 0) + s.consecutive_no_action = 2 + assert s.act(make_observation(tmp_path, 0)).kind == "cli" + with pytest.raises(ObservationRequired, match="stage_verification_returned"): + s.act(make_observation(tmp_path, 1, receipt=cli_receipt())) + assert s.consecutive_no_action == 0 + + +def test_terminal_verifier_receives_independent_stage_ledger(tmp_path): + memory = TaskMemory("Finish", requirements()) + memory.start(plan(), 0) + memory.apply(result(), {"A2": {}}) + client = Mock() + + def request(messages, validate, **kwargs): + context = json.loads(messages[1]["content"][0]["text"]) + assert context["verified_task_memory"]["work_items"][0]["status"] == "done" + assert "simultaneously" in messages[0]["content"] + return {"outcome": "achieved", "task_status": "completed", "evidence": "stage A2", + "path_input_focused": False, "state_changed": False, "requirement_checks": []} + + client.json_request.side_effect = request + make_observation(tmp_path, 0) + verify_observation(client, "Finish", tmp_path / "step_00.png", terminal_claim={"done": True}, + task_memory=verification_context(memory)) + client.json_request.assert_called_once() + + +def test_planner_gets_actual_wall_budget(tmp_path): + s = session() + s.begin("Edit", requirements()) + s.components.planner = Mock(return_value=plan()) + with pytest.raises(ObservationRequired, match="planner_returned"): + s.act(make_observation(tmp_path, 0, wall=321)) + assert s.components.planner.call_args.kwargs["remaining_seconds"] == 321 diff --git a/tests/harness/test_goal_execution.py b/tests/harness/test_goal_execution.py new file mode 100644 index 0000000..c64db49 --- /dev/null +++ b/tests/harness/test_goal_execution.py @@ -0,0 +1,236 @@ +"""Synthetic regressions derived from real failure patterns; not benchmark acceptance.""" + +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from tandem.gui.compile import gui_action +from tandem.gui.goal_candidates import foreground_tree, goal_candidates +from tandem.gui.targets import A11yTargetBuilder +from tandem.memory.context import planning_context +from tandem.models.ranking import Ranking +from tandem.planning.goal_planner import goal_error, plan_goal +from tandem.runtime.goal_stage import ( + ObservationRequired, + _action_key, + _cycle_actions, + _loop_state, + _target_effect, +) +from tandem.verification.goal_verifier import result_error + +from .conftest import A11Y_XML, build_space, build_targets, gui_receipt, make_observation +from .test_goal_memory import fact, session +from .test_goal_stage import plan, requirements, result + + +def _space(tree, inputs=()): + tree = foreground_tree(tree) + targets = A11yTargetBuilder().build(tree, "test", 128, True, True, filter_occluded=True) + return targets, goal_candidates(build_space(targets), targets, tree, inputs) + + +def test_receipt_path_detects_a_b_a_b_without_manually_seeding_attempts(tmp_path): + s = session() + s.begin("Edit", requirements()) + s.goal.memory.start(plan(), 0) + chosen = [] + + def rank(**kw): + candidates = kw["action_space"].candidates() + toggles = [c for c in candidates if c.operation == "click" and "启用" in c.description] + if len(chosen) == 4: + assert not toggles + assert kw["action_space"].controls["return_to_planner"] + winner = next(c for c in candidates if c.id == "KEYPRESS_ESCAPE") + else: + winner = toggles[0] + chosen.append(winner) + return Ranking(winner, [], [], 0, 0, 1, True) + + s.components.selector.rank = rank + for step in range(4): + tree = A11Y_XML.replace('checked="false"', f'checked="{str(step % 2 == 1).lower()}"') + s.act(make_observation(tmp_path, step, tree_xml=tree, + receipt=gui_receipt() if step else None)) + with pytest.raises(ObservationRequired, match="stage_verification_returned"): + s.act(make_observation(tmp_path, 4, tree_xml=A11Y_XML, receipt=gui_receipt())) + assert len(chosen) == 4 # The return cycle ends before offering another ineffective toggle. + assert any(e.get("end_reason") == "execution_checkpoint_stalled" for e in s.events) + assert s.goal.memory.execution_constraints + assert planning_context(s.goal.memory)["execution_constraints"] + assert s.goal.attempts[0]["state"] == s.goal.attempts[2]["state"] + + +def test_duplicate_popup_nodes_do_not_remove_their_own_close_button(): + popup = ('' + '' + '') + tree = ('' + '' + + popup * 2 + '') + targets = A11yTargetBuilder().build(tree, "test", filter_occluded=True) + assert [t.name for t in targets].count("Close") == 2 + assert "PDF" not in [t.name for t in targets] + assert "Not covered" in [t.name for t in targets] + + +def test_explicit_popup_menu_excludes_the_covered_media_button(): + tree = ('' + '' + '' + '') + ts, space = _space(tree) + assert "More" not in [t.name for t in ts] + assert any("Exit picture-in-picture" in c.description for c in space.target_heads["click"]) + + +def test_browser_memory_and_playback_badges_do_not_create_new_loop_states(): + def state(name): + tree = ('') + return _loop_state(build_targets(tree), tree) + + assert state("Video - Memory usage - 47.1 MB") == state( + "Video – Audio playing - Memory usage - 108 MB") + assert state("Different page - Memory usage - 108 MB") != state("Video - Memory usage - 47.1 MB") + + +def test_four_state_cycle_is_detected_but_continuous_scroll_is_not(): + states = ["closed", "menu", "pip", "pip-menu"] + attempts = [{"work": "w", "state": state, "key": ("click", "menu", "")} + for state in states * 2] + assert _cycle_actions(attempts, "closed") == {("click", "menu", "")} + assert not _cycle_actions([{"state": "page", "key": ("scroll", "down", "")}] * 8, "page") + + +def test_overview_excludes_stale_app_and_keeps_unnamed_search_then_typing(): + tree = ('' + '