diff --git a/README.md b/README.md
index 740d3a0..abf90a9 100644
--- a/README.md
+++ b/README.md
@@ -336,4 +336,6 @@ python -m http.server 8899 --directory demo/site # 本地预览
默认保持逐步规划;启用 `--execution-mode stage` 后,由慢模型规划阶段、快模型连续选择动作,
规则验证优先,仅在必要时调用模型验证或重规划。
+`--execution-mode goal` 可启用带任务记忆的局部连续执行,阶段结束再独立 Verify;
+保留 A11y/CDP 语义融合及 JEV 服务恢复,默认 step/stage 行为不变。
配置、执行边界、开发入口和验收方法见 [阶段执行说明](docs/STAGE_EXECUTION.md)。
diff --git a/docs/RESULT_DIR_CONTRACT.md b/docs/RESULT_DIR_CONTRACT.md
index 6867654..65839b2 100644
--- a/docs/RESULT_DIR_CONTRACT.md
+++ b/docs/RESULT_DIR_CONTRACT.md
@@ -39,6 +39,12 @@ OSWorld 2.1 显式配置替代 judge/用户模拟器时,`code_version.json.res
| `result.txt` | 分数(`score\n`) | 同上 |
| `episode.json` | `{task_id, harness, stop_reason, steps, status(scored\|failed_unscored), score, exception_type, error?, error_traceback?}` | episode 终态(必写)。`error`/`error_traceback` 仅异常题有:异常消息与栈尾几行(脱敏)——只记类型事后没法定位 |
| `timing.json` | `{wall_seconds, status}` | episode 终态 |
+| `task_memory.json`(goal) | 最新任务记忆:要求、事实、资料覆盖、交付单元、成果、失败与当前阶段 | goal 事件排水及终态 |
+| `evidence/`(goal) | 阶段验证引用的截图/CLI 全文正本;临时目录清理后仍可读取 | 阶段验收/记忆落盘 |
+
+`timing.json` 另外提供 `environment_seconds`、`model_usage_by_role`、`fast_model`,分别记录环境
+分段时间、逐角色请求时间/token、排序次数/耗时/图像使用次数。缺失 usage 不补零冒充已测值,
+另记 `calls_missing_usage`。原字段及 `browser/` 正本规则保持不变。
## 步号语义(demo 侧对齐依据)
@@ -63,7 +69,10 @@ OSWorld 2.1 显式配置替代 judge/用户模拟器时,`code_version.json.res
|------|--------|---------|
| `task_requirements` | verifier.task_requirements | episode 开始时抽取任务需求(1-12 条) |
| `plan_step` | planner.plan_step | 每步规划(截图+可见控件+需求+近期动作→下一步决策) |
-| `plan_stage` | stage_planner.plan_stage | 仅 stage 模式:初始阶段、阶段完成或请求修复时生成阶段计划 |
+| `plan_stage` | stage_planner.plan_stage / goal_planner.plan_goal | stage/goal 的阶段开始或修复;goal 允许读取已存资料后继续规划 |
+| `verify_stage` | goal_verifier.verify_goal | goal 阶段结束时独立验证或提取资料增量;明确执行失败的本地检查不算模型调用 |
+| `ground_visual` | grounding.ground_visual | goal 按需生成当前截图的有限候选,不直接执行慢模型坐标 |
+| `gui_choice` | planner_choice.PlannerChoicePolicy | 仅显式启用 planner 选择后端时,在相同候选与截图上选 ID;独立记录时间/token,不冒充 JEV 排序分数 |
| `verify_observation` | verifier.verify_observation | 每步动作执行后独立验证(BEFORE/AFTER 截图→achieved/not_achieved/uncertain)|
`verify_observation` 不信任 planner 的自我宣称:它拿前后截图+预期变化独立裁决,驱动 recent 历史、no_action 连击计数、toggle 检查、以及终态双确认(terminal_claim 模式下逐条核验 requirements + 输出文件存在性)。
@@ -80,6 +89,12 @@ OSWorld 2.1 显式配置替代 judge/用户模拟器时,`code_version.json.res
| `stage_local` | 消费已生成的阶段计划;含 stage_goal、milestone 和当帧重绑定后的 plan,不代表新的模型调用 |
| `stage_progress` | 里程碑推进,含 stage_goal、milestone、验证 source |
| `stage_replan` | 阶段修复请求,含原因及当前里程碑;target_unbound 时另含 `unbound_proposals`、`available_targets`、`recovery_scopes`;不等于调用已经完成 |
+| `task_memory` / `memory_update_rejected` | goal 的完整记忆快照/拒绝理由;不是规划模型自行宣称的完成 |
+| `goal_action` | 真实回执、当前控件、画面变化与局部效果;effect_verified=false,不等于业务验收通过 |
+| `stage_verified` | goal 的阶段裁决、证据引用、结束原因与用时,区分阶段成功和单元完成 |
+| `observation_refresh` | 刷新观察而未执行动作,action_replayed=false;不增加动作步数 |
+| `recovery` / `goal_stalled` | goal 的视觉候选补全/无可执行进展终止原因 |
+| `environment_timing` | setup/observe/execute/export/evaluate 分段耗时 |
| `action`(kind 字段) | runner 写的执行记录 `{step, kind: "action", action: {…协议动作…}}` |
stage 模式的 planner 事件只在真实规划调用时写入,增加 mode、reason、seconds;
diff --git a/infra/config.example.json b/infra/config.example.json
index a652093..d1248c1 100644
--- a/infra/config.example.json
+++ b/infra/config.example.json
@@ -44,6 +44,11 @@
"endpoint_file": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/services/qwen_jev.json",
"timeout_seconds": 300
},
+ "planner": {
+ "backend": "planner",
+ "endpoint": "",
+ "timeout_seconds": 300
+ },
"kev": {
"backend": "kev",
"endpoint": "",
@@ -63,7 +68,14 @@
},
"execution": {
"mode": "step",
- "stage_max_actions": 24
+ "stage_max_actions": 24,
+ "recovery_max_calls": 2,
+ "recovery_max_seconds": 300,
+ "verify_max_tokens": 16384,
+ "verify_reasoning_effort": null,
+ "verify_max_chars": 48000,
+ "image_quality": 85,
+ "max_request_bytes": 2097152
},
"budget": {
"max_steps": 30,
diff --git a/pyproject.toml b/pyproject.toml
index 1210a05..9024978 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,8 +7,8 @@ name = "tandem"
version = "0.1.0"
description = "Tandem:快慢双模 CUA harness(主体)+ tandem_eval 集群测评薄适配层"
requires-python = ">=3.10"
-# 运行时依赖白名单见 AGENTS.md。当前为空:M0 尚无消费者;Pillow 在 M1 截图处理落地时加入。
-dependencies = []
+# 截图处理使用仓库白名单内的 Pillow。
+dependencies = ["Pillow"]
[project.optional-dependencies]
dev = ["pytest>=7", "ruff>=0.4"]
diff --git a/scripts/run_experiment.sh b/scripts/run_experiment.sh
index 9705f10..5bce418 100755
--- a/scripts/run_experiment.sh
+++ b/scripts/run_experiment.sh
@@ -9,9 +9,12 @@ EXPERIMENT=""
TASKS=""
CONCURRENCY=4
SLOW_MODEL="kimi-k3"
+FAST_MODEL=""
SLOW_KEY_IDS="1,2,3,4"
MAX_STEPS=500
MAX_WALL=14400
+EXECUTION_MODE=stage
+STAGE_MAX_ACTIONS=10
DRY_RUN=0
usage() {
@@ -19,7 +22,8 @@ usage() {
Usage:
bash scripts/run_experiment.sh \
--experiment NAME --tasks ID1,ID2,... [--concurrency N] \
- [--slow-model NAME] [--slow-key-ids 1,2,3,4] \
+ [--slow-model NAME] [--fast-model NAME] [--slow-key-ids 1,2,3,4] \
+ [--execution-mode step|stage|goal] [--stage-max-actions N] \
[--max-steps N] [--max-wall SECONDS] [--dry-run]
Experiment names use lowercase letters, digits and hyphens only.
@@ -33,9 +37,12 @@ while [[ $# -gt 0 ]]; do
--tasks) TASKS="$2"; shift 2 ;;
--concurrency) CONCURRENCY="$2"; shift 2 ;;
--slow-model) SLOW_MODEL="$2"; shift 2 ;;
+ --fast-model) FAST_MODEL="$2"; shift 2 ;;
--slow-key-ids) SLOW_KEY_IDS="$2"; shift 2 ;;
--max-steps) MAX_STEPS="$2"; shift 2 ;;
--max-wall) MAX_WALL="$2"; shift 2 ;;
+ --execution-mode) EXECUTION_MODE="$2"; shift 2 ;;
+ --stage-max-actions) STAGE_MAX_ACTIONS="$2"; shift 2 ;;
--dry-run) DRY_RUN=1; shift ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
@@ -50,6 +57,10 @@ if [[ -z "$TASKS" || "$CONCURRENCY" -lt 1 || "$MAX_STEPS" -lt 1 || "$MAX_WALL" -
usage >&2
exit 2
fi
+if [[ ! "$EXECUTION_MODE" =~ ^(step|stage|goal)$ || "$STAGE_MAX_ACTIONS" -lt 1 ]]; then
+ usage >&2
+ exit 2
+fi
EXPERIMENT_ROOT="${TANDEM_EXPERIMENT_ROOT:-$REPO/rjob_logs}"
EXPERIMENT_DIR="$EXPERIMENT_ROOT/$EXPERIMENT"
@@ -77,6 +88,7 @@ print(json.loads(Path(sys.argv[1]).read_text())["log"]["results_root"])
PY
)
export STAGING EXPERIMENT DRIVER_DIR TASKS CONCURRENCY SLOW_MODEL SLOW_KEY_IDS MAX_STEPS MAX_WALL
+export EXECUTION_MODE STAGE_MAX_ACTIONS FAST_MODEL
python3 - <<'PY'
import json
import os
@@ -90,9 +102,12 @@ args = {
"tasks": os.environ["TASKS"].split(","),
"concurrency": int(os.environ["CONCURRENCY"]),
"slow_model": os.environ["SLOW_MODEL"],
+ "fast_model": os.environ["FAST_MODEL"] or None,
"slow_key_ids": [int(item) for item in os.environ["SLOW_KEY_IDS"].split(",")],
"max_steps": int(os.environ["MAX_STEPS"]),
"max_wall_seconds": float(os.environ["MAX_WALL"]),
+ "execution_mode": os.environ["EXECUTION_MODE"],
+ "stage_max_actions": int(os.environ["STAGE_MAX_ACTIONS"]),
"watch_timeout_seconds": float(os.environ["MAX_WALL"]) + 600,
"git_describe": subprocess.check_output(
["git", "describe", "--always", "--dirty"], text=True
@@ -123,14 +138,17 @@ run_task() {
local run_id="${EXPERIMENT}-${task}"
local driver_log="${DRIVER_DIR}/${task}.driver.log"
local rc=0
+ local fast_args=()
+ if [[ -n "$FAST_MODEL" ]]; then fast_args=(--fast-model "$FAST_MODEL"); fi
- tandem-eval rjob run \
+ env PYTHONPATH=src python3 -m tandem_eval rjob run \
--run-id "$run_id" \
--benchmark osworld21 \
--tasks "$task" \
- --execution-mode stage \
- --stage-max-actions 10 \
+ --execution-mode "$EXECUTION_MODE" \
+ --stage-max-actions "$STAGE_MAX_ACTIONS" \
--slow-model "$SLOW_MODEL" \
+ "${fast_args[@]}" \
--slow-key-ids "$SLOW_KEY_IDS" \
--max-steps "$MAX_STEPS" \
--max-wall "$MAX_WALL" \
diff --git a/src/tandem/components.py b/src/tandem/components.py
index 3df6554..4e4f785 100644
--- a/src/tandem/components.py
+++ b/src/tandem/components.py
@@ -1,6 +1,6 @@
"""组件契约(依赖注入边界,无插件框架、无隐藏分发)。
-在旧九组件上增加 semantic_enricher,统一 slow planner 与 fast selector 看到的元素语义。
-grounder(visual fallback)默认 None,session 侧判空跳过。compile_gui 产出协议动作。"""
+semantic_enricher 统一 planner 与 selector 的元素语义;goal 额外注入任务记忆、阶段验证与
+按需视觉补全。step/stage 装配不变。compile_gui 只产出协议动作,不调用 VM。"""
from __future__ import annotations
@@ -11,12 +11,17 @@
from .config import SessionConfig
from .gui.candidates import ActionCandidateBuilder
from .gui.compile import gui_action
+from .gui.grounding import ground_visual
+from .gui.interaction import GuiInteractionController
from .gui.proposals import focused_candidates
from .gui.semantics import enrich_targets
from .gui.targets import A11yTargetBuilder
+from .memory.state import TaskMemory
from .models.jev import JevOmniPolicy
+from .planning.goal_planner import goal_requirements, plan_goal
from .planning.planner import plan_step
from .planning.stage_planner import plan_stage
+from .verification.goal_verifier import verify_goal
from .verification.verifier import task_requirements, verify_observation
@@ -32,31 +37,45 @@ class Components:
focus_candidates: Callable
grounder: Callable | None
compile_gui: Callable
+ memory_factory: Callable | None = None
+ stage_verifier: Callable | None = None
+ set_deadline: Callable | None = None
+ interaction_factory: Callable = GuiInteractionController
def build_default(*, client, endpoint: str, config: SessionConfig,
image_stage: tuple[str, str] | None = None, endpoint_provider=None) -> Components:
- if config.fast_backend == "kev":
+ if config.fast_backend == "planner":
+ from .models.planner_choice import PlannerChoicePolicy
+
+ selector = PlannerChoicePolicy(client)
+ elif config.fast_backend == "kev":
from .models.kev import KevPolicy
selector = KevPolicy(endpoint, timeout_seconds=config.fast_timeout_seconds)
else:
selector = JevOmniPolicy(endpoint, timeout_seconds=config.fast_timeout_seconds,
- image_stage=image_stage, endpoint_provider=endpoint_provider)
+ image_stage=image_stage, endpoint_provider=endpoint_provider,
+ joint_actions=config.execution_mode == "goal")
return Components(
planner=partial(
+ plan_goal if config.execution_mode == "goal" else
plan_stage if config.execution_mode == "stage" else plan_step,
client,
allow_cli=config.allow_cli and config.route_policy != "gui",
route_policy=config.route_policy,
),
verifier=partial(verify_observation, client),
- requirements=partial(task_requirements, client),
+ requirements=partial(
+ goal_requirements if config.execution_mode == "goal" else task_requirements, client),
selector=selector,
target_builder=A11yTargetBuilder(),
semantic_enricher=enrich_targets,
candidate_builder=ActionCandidateBuilder(),
focus_candidates=focused_candidates,
- grounder=None,
+ grounder=partial(ground_visual, client) if config.execution_mode == "goal" else None,
compile_gui=gui_action,
+ memory_factory=TaskMemory if config.execution_mode == "goal" else None,
+ stage_verifier=partial(verify_goal, client) if config.execution_mode == "goal" else None,
+ set_deadline=getattr(client, "set_deadline", None),
)
diff --git a/src/tandem/config.py b/src/tandem/config.py
index 5786acc..9a5e36c 100644
--- a/src/tandem/config.py
+++ b/src/tandem/config.py
@@ -17,23 +17,36 @@ class SessionConfig:
cli_timeout_seconds: int = 90
execution_mode: str = "step"
stage_max_actions: int = 24
+ goal_recovery_max_calls: int = 2
+ goal_recovery_max_seconds: float = 300.0 # Checked before another grounding call, not an HTTP interrupt.
slow_max_tokens: int = 16384
+ slow_verify_max_tokens: int = 16384
+ slow_verify_reasoning_effort: str | None = None
+ stage_verify_max_chars: int = 48000
slow_timeout_seconds: float = 120
slow_direct_api: bool = False
slow_reasoning_effort: str | None = None # None=服务端默认;off=enable_thinking:false
- fast_backend: str = "jev" # "jev"(qwen-jev-choice /rank)| "kev"(Kev System One)
+ slow_image_quality: int = 85
+ slow_max_request_bytes: int = 2 * 1024 * 1024
+ fast_backend: str = "jev" # jev / kev;planner 显式复用慢模型 API 选择候选。
fast_timeout_seconds: float = 300
def __post_init__(self):
- if self.execution_mode not in {"step", "stage"}:
- raise ValueError("execution_mode must be step or stage")
+ if self.execution_mode not in {"step", "stage", "goal"}:
+ raise ValueError("execution_mode must be step, stage or goal")
if type(self.stage_max_actions) is not int or self.stage_max_actions < 1:
raise ValueError("stage_max_actions must be a positive integer")
+ if type(self.goal_recovery_max_calls) is not int or self.goal_recovery_max_calls < 1:
+ raise ValueError("goal_recovery_max_calls must be a positive integer")
+ if self.goal_recovery_max_seconds <= 0:
+ raise ValueError("goal_recovery_max_seconds must be positive")
if self.route_policy not in {"auto", "gui", "cli"}:
raise ValueError("route_policy must be auto, gui or cli")
if not 1 <= self.cli_timeout_seconds <= 120:
raise ValueError("cli_timeout_seconds must be within 1..120 (协议上限)")
- if self.fast_backend not in {"jev", "kev"}:
- raise ValueError("fast_backend must be jev or kev")
+ if self.fast_backend not in {"jev", "kev", "planner"}:
+ raise ValueError("fast_backend must be jev, kev or planner")
if self.slow_reasoning_effort not in {None, "off", "low", "medium", "high", "max"}:
raise ValueError("slow_reasoning_effort must be null/off/low/medium/high/max")
+ if self.slow_verify_reasoning_effort not in {None, "off", "low", "medium", "high", "max"}:
+ raise ValueError("slow_verify_reasoning_effort must be null/off/low/medium/high/max")
diff --git a/src/tandem/gui/candidates.py b/src/tandem/gui/candidates.py
index 2b6071a..46cbdae 100644
--- a/src/tandem/gui/candidates.py
+++ b/src/tandem/gui/candidates.py
@@ -30,8 +30,8 @@
"double_click": "The next useful step is to open one observed file or list item with a double click.",
"set_slider": "The next useful step is to set a visible slider to the required value.",
"keypress": "The next useful step is one supported keyboard navigation or submission command, not text replacement.",
- "scroll": "A required control is outside the current viewport, so scrolling is the next useful step.",
- "wait": "The required control is still loading, absent, or disabled, so waiting is the only useful next step.",
+ "scroll": "Scroll within the foreground view to reveal more controls or content.",
+ "wait": "Wait briefly, then observe the current UI again.",
"done": "Every requirement of the current local GUI goal is visibly satisfied.",
"visual_fallback": "The required interaction depends on visual or spatial information not represented by the semantic targets.",
"return_to_planner": "The current UI state contradicts the assumptions of the current stage and requires replanning.",
@@ -103,7 +103,8 @@ def build(
def relevance(target: UITarget) -> tuple[int, int]:
words = set(re.findall(r"[a-z0-9]+", target.name.lower()))
contextual_words = set(
- re.findall(r"[a-z0-9]+", " ".join((target.name, *target.parent_context)).lower())
+ re.findall(r"[a-z0-9]+", " ".join(
+ (target.name, *target.parent_context, target.nearby_text)).lower())
)
return (
len(words & hint_words) * 4 + len(contextual_words & context_words),
@@ -149,7 +150,8 @@ def candidates(operation: str, values: list[UITarget], description: str) -> list
id=f"{operation}:{target.id}",
operation=operation,
target_id=target.id,
- description=description.format(role=target.role.replace("_", " "), name=target.name),
+ description=description.format(role=target.role.replace("_", " "), name=target.name)
+ + (f" Context: {target.nearby_text}" if target.nearby_text else ""),
source=target.source,
metadata={"semantic_key": target.semantic_key},
)
diff --git a/src/tandem/gui/compile.py b/src/tandem/gui/compile.py
index c8be177..42fc37e 100644
--- a/src/tandem/gui/compile.py
+++ b/src/tandem/gui/compile.py
@@ -17,10 +17,14 @@ def gui_action(
) -> GuiAction | WaitAction | None:
operation = winner.operation
metadata = winner.metadata
- if operation in {"click", "double_click"} and target is not None:
+ if operation in {"click", "double_click", "right_click"} and (
+ target is not None or metadata.get("visual_point")
+ ):
+ point = metadata.get("visual_point") if target is None else [
+ target.x + target.width // 2, target.y + target.height // 2]
return GuiAction(
action_type=operation,
- parameters={"x": target.x + target.width // 2, "y": target.y + target.height // 2},
+ parameters={"x": point[0], "y": point[1]},
target_id=winner.target_id,
description=winner.description,
)
@@ -36,6 +40,10 @@ def gui_action(
description=winner.description,
)
if operation == "keypress":
+ if metadata.get("commit_action") and metadata.get("focus_point"):
+ return GuiAction(action_type="type", target_id=winner.target_id, description=winner.description,
+ parameters={"text": "\n", "input_mode": "insert",
+ "click": metadata["focus_point"]})
sequence = metadata.get("key_sequence") or (
[list(KEYS[winner.id])] if winner.id in KEYS else None
)
@@ -56,16 +64,22 @@ def gui_action(
description=winner.description,
)
if operation == "type_text":
- click = None
+ click = metadata.get("visual_point")
+ text = metadata["text"]
press_f2 = False
if target is not None and not metadata.get("focused_input"):
click = [target.x + target.width // 2, target.y + target.height // 2]
press_f2 = target.role in {"table_cell", "grid_cell"}
+ if target is not None and target.input_type == "date":
+ return None # Native date segments are separate observed inputs, not a tab-delimited textbox.
+ if metadata.get("choice_input"):
+ text = text.rstrip("\r\n") + "\n"
return GuiAction(
action_type="type",
parameters={
- "text": metadata["text"],
- "input_mode": metadata.get("input_mode", "replace"),
+ "text": text,
+ "input_mode": ("insert" if metadata.get("choice_input")
+ else metadata.get("input_mode", "replace")),
"click": click,
"press_f2": press_f2,
},
diff --git a/src/tandem/gui/focus.py b/src/tandem/gui/focus.py
index 60d7149..b53daca 100644
--- a/src/tandem/gui/focus.py
+++ b/src/tandem/gui/focus.py
@@ -83,7 +83,6 @@ def walk(element, active=False, menu_path=()):
["esc"],
["alt", path[0][-1].lower()],
*[[part.lower()] for part in path[1:]],
- *([["enter"]] if role == "menu_item" else []),
],
}
)
diff --git a/src/tandem/gui/goal_candidates.py b/src/tandem/gui/goal_candidates.py
new file mode 100644
index 0000000..bbce26e
--- /dev/null
+++ b/src/tandem/gui/goal_candidates.py
@@ -0,0 +1,500 @@
+"""Current-observation action space for a whole GUI subgoal; no planner click whitelist."""
+
+import json
+import re
+import xml.etree.ElementTree as ET
+from datetime import date
+from pathlib import Path
+from urllib.parse import unquote, urlsplit
+
+from .candidates import ActionCandidate
+from .focus import KEYS, input_and_menu_context
+from .targets import _attribute, _local, _pair, _state
+
+
+def matches_field(hint, label):
+ """Match an intended field to its observed label, not an arbitrary focused control."""
+ def words(value):
+ return set(re.findall(r"\w+", value.casefold())) - {"the", "a", "field", "required"}
+
+ observed = words(label)
+ intended = words(hint)
+ return (bool(observed) and (observed <= intended or len(intended) >= 2 and intended <= observed)
+ and not label.startswith(("Unnamed ", "Focused ")))
+
+
+def launcher_input(value):
+ """A launcher search is not a website's identically named Search field."""
+ return (value.get("scope") == "desktop_launcher"
+ or "desktop search" in value.get("target_hint", value["label"]).casefold())
+
+
+def _input_target(value, targets):
+ """Bind to one best observed field; spatial hints must not override its own label."""
+ scores = []
+ for target in targets:
+ choice = value.get("kind") == "choice" and target.role in {
+ "button", "push_button", "radio_button", "option", "list_item"}
+ editable = (target.role in {"textbox", "entry", "searchbox", "spin_button"}
+ or launcher_input(value) and target.role == "text" and target.focused
+ or "type_text" in target.operations or target.input_type == "select-one")
+ if not (editable or choice):
+ continue
+ if launcher_input(value) and target.application != "gnome-shell":
+ continue
+ if (target.role == "spin_button" and target.input_type != "date"
+ and re.fullmatch(r"\d{4}-\d{2}-\d{2}", value["text"])):
+ continue # A complete date is not a Day/Month/Year segment or an ordinary numeric spinner.
+ named = bool(target.name) and not target.name.startswith(("Unnamed ", "Focused "))
+ labels = [target.name] if named else [target.description]
+ if not named and len(target.nearby_text) <= 80 and "\n" not in target.nearby_text:
+ labels.append(target.nearby_text)
+ score = 0
+ if choice:
+ wanted = set(re.findall(r"\w+", value["text"].casefold()))
+ option_name = re.sub(r"\([^)]*\)", "", target.name.casefold())
+ observed = set(re.findall(r"\w+", option_name))
+ hint = set(re.findall(r"\w+", value.get("target_hint", value["label"]).casefold()))
+ if wanted and wanted <= observed <= wanted | hint:
+ score = 6 # Match a specific option, never a container holding the entire form.
+ for label in labels:
+ if not label or not editable:
+ continue
+ hint = value.get("target_hint", value["label"])
+ if not matches_field(hint, label):
+ continue # Descriptive context words cannot override the explicit intended field.
+ ignored = {"brief", "required", "enter", "input", "field", "the", "a"}
+ if (set(re.findall(r"\w+", hint.casefold())) - ignored
+ == set(re.findall(r"\w+", label.casefold())) - ignored):
+ score = max(score, 8)
+ elif matches_field(value["label"], label):
+ score = max(score, 4)
+ elif matches_field(hint, label):
+ score = max(score, 3)
+ if score:
+ scores.append((score, target))
+ best = max((score for score, _ in scores), default=0)
+ matching = [target for score, target in scores if score == best]
+ return matching[0] if len(matching) == 1 else None
+
+
+def input_satisfied(value, target):
+ """Chrome may omit a URL scheme in the displayed address, unlike ordinary text fields."""
+ text = value["text"]
+ if value.get("kind") == "choice" and target.role in {
+ "button", "push_button", "radio_button", "option", "list_item"}:
+ return target.checked is True or target.selected is True
+ if target.input_type == "select-one":
+ return target.value.strip().casefold() == text.strip().casefold()
+ if ("address" in target.name.casefold() and "://" not in target.value
+ and re.match(r"^(?:https?|file)://", text)):
+ text = text.split("://", 1)[1]
+ return target.value == text or value["input_mode"] == "insert" and text in target.value
+
+
+def input_frontier(space, targets, inputs):
+ """Expose actionable supplied inputs and missing required values on the current page."""
+ fields = list(targets)
+ pending = [c for c in space.candidates() if c.operation == "type_text"]
+ inputs = [v for v in inputs if not launcher_input(v) or _input_target(v, fields) is not None]
+ bindings = [(v, _input_target(v, fields)) for v in inputs]
+ missing = [t.name for t in fields if (t.role in {"entry", "textbox", "text", "spin_button"}
+ or t.input_type == "select-one") and not t.value
+ and re.search(r"\brequired\b|\*", t.name + " " + t.description, re.I)
+ and not any(bound is t for _, bound in bindings)]
+ unbound = [v["label"] for v, target in bindings if target is None]
+ conflicts = [t.name for t in fields
+ if len({v["text"] for v, bound in bindings if bound is t}) > 1]
+ for target in fields:
+ if target.input_type == "date" and any(
+ bound is target and not re.fullmatch(r"\d{4}-\d{2}-\d{2}", v["text"])
+ for v, bound in bindings):
+ conflicts.append(target.name + " requires ISO YYYY-MM-DD")
+ navigation = next((t for t in fields if t.focused and "address" in t.name.casefold()
+ and any(bound is t and input_satisfied(v, t)
+ and re.match(r"^(?:https?|file)://", v["text"])
+ for v, bound in bindings)), None)
+ if pending:
+ # Keep recovery/navigation choices, but do not advance past a currently unfilled supplied field.
+ space.controls.pop("done", None)
+ space.target_heads["click"] = [c for c in space.target_heads["click"] if not re.fullmatch(
+ r"(?:next(?: step)?|continue|submit(?: claim| form)?)", next(
+ (t.name for t in targets if t.id == c.target_id), ""), re.I)]
+ typed_ids = {c.target_id for c in pending if c.target_id}
+ if navigation:
+ typed_ids.add(navigation.id)
+ space.controls.pop("done", None)
+ # Type actions already focus their own observed field; a separate click adds no prerequisite.
+ space.target_heads["click"] = [c for c in space.target_heads["click"] if c.target_id not in typed_ids]
+ satisfied = [v["label"] for v, target in bindings
+ if target is not None and input_satisfied(v, target)]
+ return {"next_intent": ("Press ENTER to navigate to the supplied URL in the focused address bar."
+ if navigation else pending[0].description if pending else ""),
+ "missing_required_fields": missing, "unbound_inputs": unbound,
+ "conflicting_fields": conflicts, "satisfied_inputs": satisfied}
+
+
+def scroll_point(tree, viewport):
+ """Anchor wheel events inside the observed foreground window, not the desktop center."""
+ windows = []
+ for element in ET.fromstring(tree or "").iter():
+ if _local(element.tag) not in {"frame", "dialog"} or _state(element, "active") is not True:
+ continue
+ position = _pair(_attribute(element, "screencoord"))
+ size = _pair(_attribute(element, "size"))
+ if position and size and size[0] > 0 and size[1] > 0:
+ windows.append((*position, *size))
+ x, y, width, height = windows[-1] if windows else (0, 0, *viewport)
+ return (max(0, min(viewport[0] - 1, x + width * 3 // 5)),
+ max(0, min(viewport[1] - 1, y + height // 2)))
+
+
+def _live_dialog(node, root):
+ """Unknown visibility on a retained portal node is not proof of a blocking dialog."""
+ if (_state(node, "modal") is not True or _state(node, "visible") is False
+ or _state(node, "showing") is False):
+ return False
+ positioned = any(_pair(_attribute(e, "screencoord")) and
+ (size := _pair(_attribute(e, "size"))) and min(size) > 1 for e in node.iter())
+ return bool(positioned and (_state(node, "showing") is True
+ or _state(node, "active") is True
+ or node.get("name") == root.get("foreground_title")))
+
+
+def _keys(candidate):
+ return candidate.metadata.get("key_sequence") or (
+ [list(KEYS[candidate.id])] if candidate.id in KEYS else None)
+
+
+def portal_needs_grounding(tree):
+ """Retained coordinate-free portal nodes require screenshot confirmation, not background clicks."""
+ root = ET.fromstring(tree or "")
+ return next((node.get("name", "") for app in root.iter() if "desktop-portal" in app.get("name", "")
+ for node in app.iter() if _local(node.tag) == "dialog" and len(node) > 0
+ and _state(node, "modal") is True and _state(node, "visible") is not False
+ and _state(node, "showing") is not False and not _live_dialog(node, root)), "")
+
+
+def foreground_tree(raw_tree):
+ """GNOME's visible app menu disambiguates stale ACTIVE flags on background windows."""
+ root = ET.fromstring(raw_tree)
+ apps = list(root)
+ shell = next((a for a in apps if a.get("name") == "gnome-shell"), None)
+ if shell is None:
+ return raw_tree
+ overview = [e for e in shell.iter() if e.get("name") == "Overview"]
+ search_visible = any(
+ _state(e, "editable") is True and _state(e, "showing") is True
+ and _pair(_attribute(e, "screencoord")) and _pair(_attribute(e, "size"))
+ for panel in overview for e in panel.iter())
+ if search_visible:
+ # X11 still names the old window while GNOME's full-screen overview owns input.
+ for app in apps:
+ if app is not shell:
+ root.remove(app)
+ shell.set("active", "true")
+ root.set("foreground_title", "GNOME Overview")
+ root.set("foreground_class", '["gnome-shell"]')
+ return ET.tostring(root, encoding="unicode")
+ # A portal dialog belongs to a separate process; stale X11 focus must not discard it.
+ portals = [(app, node) for app in apps if "desktop-portal" in app.get("name", "")
+ for node in app.iter() if _local(node.tag) == "dialog"
+ and _live_dialog(node, root)]
+ if portals:
+ app, dialog = portals[-1]
+ app.set("active", "true")
+ root.set("foreground_title", dialog.get("name", ""))
+ root.set("foreground_class", json.dumps([app.get("name", "")]))
+ if root.get("foreground_class"):
+ def normalize(value):
+ return re.sub(r"[^a-z0-9]", "", value.casefold())
+
+ def active(element, value):
+ key = next((k for k in element.attrib if _local(k) == "active"), "active")
+ element.set(key, "true" if value else "false")
+
+ classes = [normalize(v) for v in json.loads(root.get("foreground_class")) if v]
+ title = root.get("foreground_title", "")
+ for app in apps:
+ matching = (any(normalize(app.get("name", "")).endswith(c) for c in classes)
+ or bool(title) and any(e.get("name") == title for e in app.iter()
+ if _local(e.tag) in {"frame", "dialog"}))
+ uncertain_portal = "desktop-portal" in app.get("name", "") and portal_needs_grounding(
+ ET.tostring(app, encoding="unicode"))
+ if app is not shell and not matching and not uncertain_portal:
+ root.remove(app)
+ elif app is not shell:
+ windows = [e for e in app.iter() if _local(e.tag) in {"frame", "dialog"}]
+ named = [e for e in windows if title and e.get("name") == title]
+ if named:
+ active(app, False)
+ for window in windows:
+ active(window, window in named)
+ return ET.tostring(root, encoding="unicode")
+ names = []
+ for element in shell.iter():
+ attrs = {k.rsplit("}", 1)[-1]: v for k, v in element.attrib.items()}
+ position = re.findall(r"-?\d+", attrs.get("screencoord", ""))
+ if (element.tag.rsplit("}", 1)[-1] == "menu" and attrs.get("name")
+ and attrs.get("showing") == "true" and len(position) == 2
+ and 0 <= int(position[1]) < 32):
+ names.append(attrs["name"].casefold())
+ matches = [app for app in apps if app is not shell and any(
+ name in e.get("name", "").casefold() for e in app.iter()
+ if e.tag.rsplit("}", 1)[-1] in {"frame", "dialog"} for name in names)]
+ if len(matches) != 1:
+ return raw_tree
+ for app in apps:
+ if app is not shell and app is not matches[0]:
+ root.remove(app)
+ return ET.tostring(root, encoding="unicode")
+
+
+def goal_candidates(space, targets, tree, inputs, viewport=(1920, 1080)):
+ _, menus = input_and_menu_context(tree or "")
+ fields = list(targets)
+ bindings = [_input_target(value, fields) for value in inputs]
+ typed = []
+ for index, value in enumerate(inputs):
+ hint = value.get("target_hint", value["label"])
+ target = bindings[index]
+ if target is None:
+ continue # Ambiguous/unlabelled fields need an observed click or visual grounding first.
+ if len({v["text"] for v, bound in zip(inputs, bindings, strict=True) if bound is target}) > 1:
+ continue
+ if input_satisfied(value, target):
+ continue
+ if value.get("kind") == "choice" and target.role in {
+ "button", "push_button", "radio_button", "option", "list_item"}:
+ space.target_heads["click"].insert(0, ActionCandidate(
+ id=f"GOAL_CHOICE:{index}", operation="click", source="goal_input", target_id=target.id,
+ description=f"Select the supplied option {value['text']!r} on {target.name!r}.",
+ metadata={"semantic_key": target.semantic_key}))
+ continue
+ if target.input_type == "date":
+ try:
+ if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value["text"]):
+ continue
+ date.fromisoformat(value["text"])
+ except ValueError:
+ continue # Ask the planner for an unambiguous date rather than guessing its order.
+ if len(target.date_order) != 3 or set(target.date_order) != {"year", "month", "day"}:
+ continue
+ parts = [t for t in fields if t.role == "spin_button"
+ and t.name.casefold().split(" ", 1)[0] in target.date_order
+ and abs(t.y - target.y) <= 8 and target.x - 8 <= t.x < target.x + target.width]
+ if len(parts) != 3 or {t.name.casefold().split(" ", 1)[0] for t in parts} != {
+ "year", "month", "day"}:
+ continue
+ expected = dict(zip(("year", "month", "day"), value["text"].split("-"), strict=True))
+ target = next((t for part in ("year", "month", "day") for t in parts
+ if t.name.casefold().split(" ", 1)[0] == part and (not t.value.isdigit()
+ or int(t.value) != int(expected[part]))), None)
+ if target is None:
+ continue
+ value = {**value, "text": expected[target.name.casefold().split(" ", 1)[0]]}
+ hint = target.name
+ typed.append(ActionCandidate(
+ id=f"GOAL_TEXT:{index}", operation="type_text", source="goal_input", target_id=target.id,
+ description=(f"Select {value['text']!r} in the observed native choice {target.name!r} "
+ "using keyboard prefix and Enter." if target.input_type == "select-one" else
+ f"{value['input_mode']} the observed field {target.name!r} with the "
+ f"supplied {value['label']}: {value['text']!r}"),
+ metadata={"text": value["text"], "input_mode": value["input_mode"],
+ "choice_input": target.input_type == "select-one",
+ "target_hint": hint, "field_label": target.name,
+ "semantic_key": target.semantic_key},
+ ))
+ space.target_heads["type_text"] = typed
+ unnamed_entries = [t for t in targets if t.name.startswith(("Unnamed ", "Focused "))
+ and t.role in {"text", "textbox", "entry", "searchbox"}][:2]
+ present = {c.target_id for c in space.target_heads["click"]}
+ space.target_heads["click"] = [ActionCandidate(
+ id="FOCUS:" + t.id, operation="click", target_id=t.id, source="a11y",
+ description=f"Focus the observed {t.name!r} in {' / '.join(t.parent_context)} before typing.",
+ metadata={"semantic_key": t.semantic_key}) for t in unnamed_entries if t.id not in present
+ ] + space.target_heads["click"]
+ native_files = [t for t in targets if t.role in {"canvas", "label"}
+ and "nautilus" in (t.application + " ".join(t.parent_context)).lower()][:4]
+ space.target_heads["double_click"].extend(ActionCandidate(
+ id="OPEN:" + t.id, operation="double_click", target_id=t.id, source=t.source,
+ description=f"Open the observed file-manager item {t.name!r}.",
+ metadata={"semantic_key": t.semantic_key}) for t in native_files)
+ sliders = [t for t in targets if t.role == "slider"][:2]
+ space.target_heads["set_slider"] = [ActionCandidate(
+ id=f"SLIDER:{t.id}:{fraction}", operation="set_slider", target_id=t.id,
+ description=f"Set {t.name!r} to {fraction:.0%} of its range"
+ + (f" (value {t.range_min + fraction * (t.range_max - t.range_min):.2f})"
+ if t.range_min is not None and t.range_max is not None else "")
+ + f"; current observed value: {t.value or 'unknown'}.",
+ metadata={"fraction": fraction, "semantic_key": t.semantic_key})
+ for t in sliders for fraction in (0.1, 0.25, 0.5, 0.75, 0.9)
+ if (not t.value or t.range_min is None or t.range_max is None
+ or abs(float(t.value) - (t.range_min + fraction * (t.range_max - t.range_min)))
+ > max(1e-6, (t.range_max - t.range_min) * 0.02))]
+ # A slider's default center click is ambiguous and was repeatedly seeking to the same midpoint.
+ slider_ids = {t.id for t in sliders}
+ space.target_heads["click"] = [c for c in space.target_heads["click"] if c.target_id not in slider_ids]
+ space.target_heads["right_click"] = [ActionCandidate(
+ id="RIGHT:" + c.target_id, operation="right_click", source=c.source, target_id=c.target_id,
+ description="Open the context menu for " + c.description,
+ metadata=dict(c.metadata)) for c in space.target_heads["click"][:8]]
+ sx, sy = scroll_point(tree, viewport)
+ for candidate in space.target_heads["scroll"]:
+ candidate.metadata.update(scroll_x=sx, scroll_y=sy)
+ for menu in menus[:16]:
+ space.target_heads["keypress"].append(ActionCandidate(
+ id="MENU_KEY:" + menu["path"], operation="keypress", source="a11y_menu",
+ description="Open the observed menu path " + menu["path"],
+ metadata={"key_sequence": menu["key_sequence"]}))
+ space.target_heads["keypress"].extend(ActionCandidate(
+ id="GOAL_KEY:" + "+".join(keys), operation="keypress", source="global",
+ description=description, metadata={"key_sequence": [keys]}) for keys, description in [
+ (["ctrl", "s"], "Save the current document."),
+ (["ctrl", "o"], "Open the application's file chooser for an existing document."),
+ (["alt", "tab"], "Switch away from the foreground window to another open application."),
+ (["shift", "f10"], "Open the focused object's context menu."),
+ (["ctrl", "l"], "Focus the browser or native file-dialog location field."),
+ (["ctrl", "f"], "Open the current application's Find field."),
+ (["shift", "tab"], "Move focus to the previous field."),
+ (["down"], "Move down in the current selection or menu."),
+ (["up"], "Move up in the current selection or menu."),
+ (["right"], "Move right in the current selection, calendar or menu."),
+ (["left"], "Move left in the current selection, calendar or menu."),
+ (["ctrl", "t"], "Open a separate browser tab, preserving the current tab."),
+ (["ctrl", "home"], "Move to the beginning of the current document or page."),
+ (["ctrl", "end"], "Move to the end of the current document or page."),
+ (["f6"], "Move keyboard focus to the next pane."),
+ (["space"], "Select or activate the currently focused control."),
+ ])
+ title = ET.fromstring(tree or "").get("foreground_title", "")
+ if re.match(r"New Tab(?:\s*[-–]|$)", title, re.I) and any(
+ "address" in t.name.casefold() and _input_target(v, fields) is t
+ for t in fields for v in inputs):
+ space.target_heads["click"] = [c for c in space.target_heads["click"] if not any(
+ t.id == c.target_id and t.scope == "browser_chrome" and t.name.casefold() == "new tab"
+ for t in targets)]
+ space.controls = {k: v for k, v in space.controls.items()
+ if k in {"wait", "done", "visual_fallback", "return_to_planner"}}
+ space.controls["done"] = ActionCandidate(
+ id="STAGE_DONE", operation="done", source="stage_control",
+ description="The current subtask's completion criteria are satisfied. Request ONE independent "
+ "stage verification; this does not finish the original task.")
+ space.controls["return_to_planner"].description = (
+ "The current subtask needs unavailable information, new reasoning or an unsupported operation. "
+ "Return control with existing progress preserved; do not repeat an ineffective action.")
+ return space
+
+
+def modal_frontier(space, targets, tree):
+ """A real blocking dialog owns interaction; background movement is not useful recovery."""
+ root = ET.fromstring(tree or "")
+ dialogs = [n for n in root.iter() if _local(n.tag) in {"dialog", "alert"} and _live_dialog(n, root)]
+ visual = next((c.metadata for c in space.candidates() if c.metadata.get("blocking_overlay")), {})
+ if not dialogs and not visual:
+ return {}
+ dialog = dialogs[-1] if dialogs else None
+ name = dialog.get("name", "Dialog") if dialog is not None else visual["overlay_name"]
+ position = _pair(_attribute(dialog, "screencoord")) if dialog is not None else None
+ size = _pair(_attribute(dialog, "size")) if dialog is not None else None
+ bounds = [*position, *size] if position and size else visual.get("overlay_bounds")
+ by_id = {t.id: t for t in targets}
+
+ def inside(candidate):
+ target = by_id.get(candidate.target_id)
+ focus = candidate.metadata.get("focus_point")
+ if focus and bounds:
+ x, y, width, height = bounds
+ if not (x <= focus[0] < x + width and y <= focus[1] < y + height):
+ return False
+ if candidate.operation == "keypress":
+ return _keys(candidate) in [
+ [[k]] for k in ("esc", "escape", "enter", "tab", "up", "down", "left", "right", "f6", "space")
+ ] + [[["ctrl", "l"]], [["shift", "tab"]]]
+ if bounds:
+ point = candidate.metadata.get("visual_point")
+ if target:
+ point = [target.x + target.width // 2, target.y + target.height // 2]
+ if candidate.operation == "scroll":
+ point = [candidate.metadata["scroll_x"], candidate.metadata["scroll_y"]]
+ x, y, width, height = bounds
+ return bool(point and x <= point[0] < x + width and y <= point[1] < y + height)
+ return bool(candidate.metadata.get("blocking_overlay") or target and name in target.parent_context)
+
+ for operation in space.target_heads:
+ space.target_heads[operation] = [c for c in space.target_heads[operation] if inside(c)]
+ space.controls = {k: c for k, c in space.controls.items()
+ if k in {"return_to_planner", "visual_fallback"}}
+ return {"blocking_overlay": name, "overlay_bounds": bounds,
+ "overlay_kind": visual.get("overlay_kind", "unknown"),
+ "next_intent": f"The active dialog {name!r} owns input; work inside its visible controls."}
+
+
+def document_mismatch(document, tree, targets):
+ """Reject observed contradictions, not a website's human title or a generic app title."""
+ title = ET.fromstring(tree or "").get("foreground_title", "")
+ filename = Path(document).name.casefold()
+ if Path(document).stem.casefold() in title.casefold():
+ return False
+ addresses = [t.value for t in targets if "address" in t.name.casefold() and "://" in t.value]
+ if Path(document).suffix.casefold() in {".html", ".htm"} and addresses:
+ return all(Path(unquote(urlsplit(v).path)).name.casefold() != filename for v in addresses)
+ return bool(re.search(r"\buntitled\b|\S+\.(?:pptx?|docx?|xlsx?|od[ptsg]|pdf)\b", title, re.I))
+
+
+def application_mismatch(application, tree, known_only=False):
+ """Only positive app-family contradictions gate editing; ambiguous titles remain unknown."""
+ root = ET.fromstring(tree or "")
+ actual = (root.get("foreground_title", "") + " " + root.get("foreground_class", "")).casefold()
+ families = ({"wps", "wpp", "et"}, {"libreoffice", "soffice"}, {"chrome", "chromium"},
+ {"shotcut"}, {"totem", "videos"}, {"nautilus", "files"})
+ wanted = set(re.findall(r"\w+", application.casefold()))
+ seen = set(re.findall(r"\w+", actual))
+ desired = next((f for f in families if f & wanted), None)
+ current = next((f for f in families if f & seen), None)
+ if desired and current:
+ return desired != current
+ tokens = [w for w in wanted if len(w) >= 4 and w not in {
+ "classic", "presentation", "office", "desktop", "google"}]
+ if any(w in actual for w in tokens):
+ return False
+ observed = root.get("foreground_class", "").casefold()
+ return bool(tokens and (current or observed and not known_only and not any(
+ name in observed for name in ("gnome-shell", "desktop-portal"))))
+
+
+def document_frontier(space, targets, document=None, application=None):
+ """Permit opening the right document, never typing task content into a contradicted one."""
+ by_id = {t.id: t for t in targets}
+ filename = Path(document).name.casefold() if document else ""
+
+ def navigation(candidate):
+ target = by_id.get(candidate.target_id)
+ if candidate.operation == "type_text":
+ text = candidate.metadata.get("text", "")
+ return bool(target and (application and target.application == "gnome-shell"
+ and text.casefold() == application.casefold()
+ or filename and ("address" in target.name.casefold() or target.scope == "dialog")
+ and Path(unquote(urlsplit(text).path)).name.casefold() == filename))
+ if candidate.operation == "keypress":
+ return _keys(candidate) in [
+ [["ctrl", "l"]], [["ctrl", "o"]], [["esc"]], [["escape"]], [["alt", "tab"]]
+ ] or candidate.id == "KEYPRESS_ENTER" and any(
+ t.focused and ("address" in t.name.casefold() or t.scope == "dialog"
+ or t.application == "gnome-shell") for t in targets)
+ if candidate.source == "visual_grounder" and candidate.operation in {"click", "double_click"}:
+ label = candidate.metadata.get("target_label", "").casefold()
+ return bool(filename and filename in label or application and matches_field(application, label))
+ return bool(target and candidate.operation in {"click", "double_click"} and (
+ target.scope in {"browser_chrome", "desktop_launcher"}
+ or target.application == "gnome-shell"
+ or filename and filename in target.name.casefold()
+ or application and application.casefold() in target.name.casefold()))
+
+ for operation in space.target_heads:
+ space.target_heads[operation] = [c for c in space.target_heads[operation] if navigation(c)]
+ space.controls = {k: c for k, c in space.controls.items()
+ if k in {"return_to_planner", "visual_fallback"}}
+ return {"document_recovery": document, "application_recovery": application,
+ "next_intent": f"Navigate to {application or ''} {document or ''} first. "
+ "Do not edit the wrong application/document."}
diff --git a/src/tandem/gui/grounding.py b/src/tandem/gui/grounding.py
new file mode 100644
index 0000000..1e99714
--- /dev/null
+++ b/src/tandem/gui/grounding.py
@@ -0,0 +1,86 @@
+"""Existing screenshot-grounded target fallback, reused without changing the JEV service."""
+
+import base64
+import json
+import re
+
+from PIL import Image
+
+from .candidates import ActionCandidate
+from .goal_candidates import matches_field
+
+
+def ground_visual(client, instruction, plan, screenshot):
+ with Image.open(screenshot) as image:
+ width, height = image.size
+ proposals = plan["action_proposals"]
+ value = client.json_request([
+ {"role": "system", "content": (
+ "Ground the immediate GUI intent in the CURRENT screenshot. Propose up to four distinct "
+ "visible target alternatives, never a screen grid or a sequence. Do not execute anything. "
+ "Return JSON {blocking_overlay:null|{name,bounds,kind},"
+ "candidates:[{operation,target,bounds,text,input_mode,target_hint}]}. "
+ "If a modal or file chooser blocks the task, identify it and propose ONLY its controls. "
+ "kind=task_dialog when it is the requested interaction (save, open, choose); "
+ "kind=obstruction only for an unrelated prompt preventing the local objective; "
+ "otherwise kind=unknown. Do not close a task dialog as a generic recovery. "
+ "Do not label an ordinary document/window as a blocking overlay. "
+ "Operations: click, double_click, right_click, type_text, scroll_up, scroll_down. "
+ "bounds=[x,y,width,height] in "
+ "ORIGINAL screenshot pixels. Ground only foreground unobscured targets. For type_text copy exact "
+ "text/input_mode AND target_hint from ONE supplied input, and ground the visible field named "
+ "by that hint. target must be its observed label/identity, not the text to enter. Never put "
+ "unrelated text in the currently focused field; otherwise click to focus first. Use [] if no "
+ "For date/calendar fields, propose opening the calendar, NOT whole-string type_text: "
+ "the screenshot alone does not establish the native date input's segment order. "
+ "required target is visible. Do not invent targets or completion evidence."
+ )},
+ {"role": "user", "content": [
+ {"type": "text", "text": json.dumps({"task": instruction, "local_plan": plan,
+ "screenshot_size": [width, height]}, ensure_ascii=False)},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,"
+ + base64.b64encode(screenshot.read_bytes()).decode("ascii")}},
+ ]},
+ ], lambda v: isinstance(v.get("candidates"), list)
+ and all(isinstance(i, dict) for i in v["candidates"]), role="ground_visual")
+ allowed_text = {(p.get("text"), p.get("input_mode"), p.get("target_hint", p.get("label", "")))
+ for p in proposals if p.get("operation") == "type_text"}
+ candidates = []
+ for item in value["candidates"][:4]:
+ operation, box = item.get("operation"), item.get("bounds")
+ if operation not in {"click", "double_click", "right_click", "type_text", "scroll_up", "scroll_down"}:
+ continue
+ if not isinstance(box, list) or len(box) != 4 or any(type(v) is not int for v in box):
+ continue
+ x, y, w, h = box
+ if not (0 <= x < x + w <= width and 0 <= y < y + h <= height):
+ continue
+ point = [x + w // 2, y + h // 2]
+ metadata = {"visual_point": point, "visual_bounds": box, "target_label": item.get("target", "")}
+ overlay = value.get("blocking_overlay")
+ if isinstance(overlay, dict) and isinstance(overlay.get("name"), str):
+ bounds = overlay.get("bounds")
+ if (isinstance(bounds, list) and len(bounds) == 4 and all(type(v) is int for v in bounds)
+ and 0 <= bounds[0] < bounds[0] + bounds[2] <= width
+ and 0 <= bounds[1] < bounds[1] + bounds[3] <= height):
+ metadata.update(blocking_overlay=True, overlay_name=overlay["name"], overlay_bounds=bounds)
+ metadata["overlay_kind"] = overlay.get("kind", "unknown")
+ if operation == "type_text":
+ if (re.fullmatch(r"\d{4}-\d{2}-\d{2}", str(item.get("text", "")))
+ and re.search(r"date|time|calendar|日期|时间", str(item.get("target_hint", "")), re.I)):
+ continue # Native date typing requires observed CDP input_type/date_order.
+ if ((item.get("text"), item.get("input_mode"), item.get("target_hint")) not in allowed_text
+ or not matches_field(item.get("target_hint", ""), item.get("target", ""))):
+ continue
+ metadata.update(text=item["text"], input_mode=item["input_mode"],
+ target_hint=item["target_hint"], field_label=item["target"])
+ name = f"VISUAL:{operation}:{x}:{y}:{w}:{h}"
+ if operation.startswith("scroll_"):
+ name = f"{operation.upper()}:{name}"
+ operation = "scroll"
+ metadata.update(scroll_x=point[0], scroll_y=point[1])
+ candidates.append(ActionCandidate(
+ id=name, operation=operation, source="visual_grounder", metadata=metadata,
+ description=f"{item['operation']} screenshot-visible {item.get('target', 'control')!r} "
+ f"at box {box}." + (f" Text: {metadata['text']!r}." if "text" in metadata else "")))
+ return candidates
diff --git a/src/tandem/gui/interaction.py b/src/tandem/gui/interaction.py
new file mode 100644
index 0000000..187cbbd
--- /dev/null
+++ b/src/tandem/gui/interaction.py
@@ -0,0 +1,452 @@
+"""Host-owned field/commit state; no model calls, VM execution or semantic success claims."""
+
+import json
+import re
+import xml.etree.ElementTree as ET
+from copy import deepcopy
+from pathlib import Path
+
+from PIL import Image, ImageChops, ImageStat
+
+from .candidates import ActionCandidate
+from .goal_candidates import (
+ _input_target,
+ _keys,
+ application_mismatch,
+ document_frontier,
+ input_frontier,
+ input_satisfied,
+ launcher_input,
+ matches_field,
+ modal_frontier,
+ portal_needs_grounding,
+)
+from .semantics import unrepresented_window
+
+
+def visual_changed(before, after, bounds=None):
+ frames, sizes = [], []
+ for path in (before, after):
+ with Image.open(path) as image:
+ sizes.append(image.size)
+ box = ((bounds[0], bounds[1], bounds[0] + bounds[2], bounds[1] + bounds[3]) if bounds
+ else (0, min(32, image.height // 4), image.width, image.height))
+ frames.append(image.crop(box).convert("L").resize((160, 90)))
+ threshold = 0.1 if bounds else 1.0
+ return sizes[0] != sizes[1] or ImageStat.Stat(ImageChops.difference(*frames)).mean[0] > threshold
+
+
+def visual_identity(candidate):
+ """A field's input attempt does not become new because its detected box shifts a few pixels."""
+ if candidate.source == "visual_grounder" and candidate.operation == "type_text":
+ return "visual_input|" + " ".join(candidate.metadata.get("target_hint", "").casefold().split())
+ return None
+
+
+def _binding(value):
+ return json.dumps([value.get("target_hint", value["label"]).casefold(), value["text"].rstrip()],
+ ensure_ascii=False)
+
+
+def _command_input(value):
+ return value.get("kind") == "command" or value.get("commit") == "enter" or bool(
+ re.search(r"\b(input|command|console)\b", value.get("target_hint", value["label"]), re.I)
+ and re.match(r"\w+\s*\(", value["text"]))
+
+
+def target_effect(old, targets):
+ """Observed field/slider effects, never semantic task acceptance."""
+ if old["operation"] == "type_text":
+ focus = [t for t in targets if t.focused and t.role != "password_text"
+ and (not old.get("field_label") or matches_field(old["field_label"], t.name))]
+ intended = [t for t in targets if old.get("target_semantic")
+ and old["target_semantic"] == t.semantic_key]
+ if not intended and old.get("field_label"):
+ intended = [t for t in targets if matches_field(old["field_label"], t.name)
+ and t.role in {"entry", "text", "textbox", "spin_button", "searchbox"}]
+ if intended:
+ focus = intended
+ if len(focus) == 1 and (focus[0].value or old.get("field_label")):
+ if old.get("field_label") and not matches_field(old["field_label"], focus[0].name):
+ return "not_met"
+ matches = input_satisfied(
+ {"text": old["expected_text"], "input_mode": old["input_mode"]}, focus[0])
+ return "met" if matches else "not_met"
+ if old["operation"] == "set_slider":
+ target = next((t for t in targets if t.semantic_key == old["target_semantic"]), None)
+ if (target and target.value and target.range_min is not None and target.range_max is not None
+ and target.range_max > target.range_min):
+ fraction = (float(target.value) - target.range_min) / (target.range_max - target.range_min)
+ return "met" if abs(fraction - old["fraction"]) <= 0.04 else "not_met"
+ return "unknown"
+
+
+def _content_state(targets):
+ """Task controls/values, not focus, coordinates, shell clocks or pixel animation."""
+ return tuple(sorted((t.role, re.sub(r" - Memory usage - .+$", "", t.name),
+ "" if t.role == "slider" else t.value, str(t.checked)) for t in targets
+ if t.application != "gnome-shell" and "gnome-shell" not in t.parent_context
+ and not t.name.startswith(("Focused ", "Unnamed "))))
+
+
+def _navigation_family(operation, sequence):
+ if operation == "scroll":
+ return "scroll"
+ if sequence in ([["tab"]], [["shift", "tab"]]):
+ return "focus_traversal"
+ if sequence in ([["up"]], [["down"]], [["left"]], [["right"]]):
+ return "direction_keys"
+ if sequence in ([["enter"]], [["esc"]], [["escape"]]):
+ return "activation_keys"
+ return None
+
+
+class GuiInteractionController:
+ def __init__(self):
+ self.stage_id = None
+ self.fields = {}
+ self.invalidate_visual = False
+ self.control = None
+ self.checkpoint_due = False
+ self.local_effect = "not_assessed"
+ self.content_state = ()
+ self.scenes = {}
+ self.memory = None
+ self.observation_ref = ""
+ self.rejected_states = {}
+
+ def receive(self, pending, effect, changed, returncode, *, targets=(), evidence_ref=""):
+ """An acknowledged dispatch is not proof of the field value or command's success."""
+ self.local_effect = "not_assessed"
+ key = pending.get("interaction_binding")
+ if key in self.fields:
+ field = self.fields[key]
+ if returncode:
+ field["phase"] = "needs_input"
+ elif pending.get("commit_action"):
+ field["phase"] = "submitted_unconfirmed"
+ elif effect == "not_met":
+ field["phase"] = "needs_input"
+ elif field["commit"]:
+ field["phase"] = "needs_commit"
+ else:
+ field["phase"] = "confirmed" if effect == "met" else "typed_unconfirmed"
+ field.update(observed_change=changed, effect=effect)
+ sequence = pending.get("key_sequence")
+ self.invalidate_visual = (pending["operation"] == "scroll" or sequence in (
+ [["alt", "tab"]], [["alt", "f4"]], [["ctrl", "o"]], [["ctrl", "l"]]))
+ if self.control is None or pending.get("control_scope") != self.control["scope_key"]:
+ return
+ current = _content_state(targets)
+ scenes = self.scenes.setdefault(self.control["scope_key"], [pending["control_state"]])
+ revisited = current in scenes
+ scenes.append(current)
+ del scenes[:-16]
+ local = not returncode and effect != "not_met" and (
+ effect == "met" or current != pending["control_state"])
+ self.local_effect = "confirmed_input" if effect == "met" else (
+ "controls_changed" if local else "unknown" if changed else "no_observed_effect")
+ if self.memory is not None:
+ blocker = self.memory.active_blocker
+ if (blocker and blocker["owner"] == self.control["scope_key"]
+ and blocker["kind"] in {"rejected_completion", "no_effect"}
+ and (effect == "met" or local and not revisited
+ or pending.get("visual_action") and changed)):
+ self.memory.resolve_blocker(blocker["owner"], evidence_ref)
+ self.control.pop("completion_rejected", None)
+ elif not local and (self.control["without_effect"] or returncode):
+ self.memory.block("no_effect", self.control["scope_key"],
+ "The last interaction did not advance the delegated objective", evidence_ref,
+ attempt={"action": pending["operation"], "result": "no_effect"})
+ if not returncode and effect == "met":
+ self.control.update(attempts={}, without_effect=0, visited_focus=[], ground_calls=0,
+ ground_seconds=0.0)
+ self.checkpoint_due = False
+ return
+ if local and not revisited:
+ if not self.control["recovery_required"]:
+ self.control.update(ground_calls=0, ground_seconds=0.0)
+ self.checkpoint_due = False
+ return # New content permits exploration, but does not release previously failed paths.
+ self.control["without_effect"] += 1
+ sequence = pending["action"].get("parameters", {}).get("sequence")
+ family = pending.get("execution_family") or _navigation_family(pending["operation"], sequence)
+ if family and not pending.get("commit_action"):
+ attempts = self.control["attempts"]
+ attempts[family] = attempts.get(family, 0) + 1
+ focused = [t.semantic_key or t.name for t in targets if t.focused]
+ focus = "|".join(focused)
+ visited = self.control["visited_focus"]
+ cycle = bool(focus and focus in visited)
+ if family == "focus_traversal" and focus and focus not in visited:
+ visited.append(focus)
+ self.checkpoint_due = False # Visiting another known field is finite exploration.
+ else:
+ self.checkpoint_due = attempts[family] >= (3 if family.startswith("target:") else 4) and (
+ family != "focus_traversal" or not focus or cycle)
+ elif self.control["without_effect"] >= 8:
+ self.checkpoint_due = True
+
+ def reusable_visual(self, frame, state, screenshot):
+ if not frame:
+ return False
+ return (not self.invalidate_visual and frame["state"] == state
+ and not visual_changed(frame["screenshot"], screenshot))
+
+
+ def finalize(self, space, targets, stage, memory, tree, observation_ref=""):
+ """All producers run before the final scope/failed-path checks. No action executes here."""
+ self.memory, self.observation_ref = memory, observation_ref
+ def scope():
+ result = modal_frontier(space, targets, tree)
+ if result:
+ return result # Resolve the actual input owner before recovering an underlying document.
+ if stage.get("document_recovery") or stage.get("application_recovery"):
+ result.update(document_frontier(space, targets, stage.get("target_document"),
+ stage.get("target_application")))
+ return result
+
+ scope()
+ interaction = self.constrain(space, targets, stage, memory, tree=tree)
+ limits = scope() # Commit candidates must obey the same scope as every other producer.
+ root = ET.fromstring(tree or "")
+ overlay = limits.get("blocking_overlay", "")
+ is_dialog = "blocking_overlay" in limits
+ bounds = limits.get("overlay_bounds")
+ owner = portal_needs_grounding(tree) or unrepresented_window(root) or (
+ "dialog@" + str([round(v / 32) for v in bounds]) if bounds else overlay)
+ key = json.dumps([root.get("foreground_class", ""), root.get("foreground_title", ""),
+ owner], ensure_ascii=False)
+ blocker = memory.active_blocker
+ if (blocker and blocker["status"] != "resolved" and blocker["kind"] == "input_obstructed"
+ and not is_dialog and json.loads(blocker["owner"])[0] == root.get("foreground_class", "")):
+ memory.resolve_blocker(blocker["owner"], observation_ref)
+ for old in memory.execution_constraints:
+ if old.get("scope_key") == blocker["owner"]:
+ old.update(recovery_required=False, blocked_families=[],
+ ground_calls=0, ground_seconds=0.0)
+ memory.work_items[blocker["work_item_id"]]["recovery_required"] = False
+ control = next((c for c in memory.execution_constraints
+ if c.get("kind") == "execution_checkpoint" and c["scope_key"] == key), None)
+ if control is None:
+ control = {"kind": "execution_checkpoint", "scope_key": key,
+ "work_item_id": stage["work_item_id"], "global_scope": bool(owner),
+ "attempts": {}, "blocked_families": [], "without_effect": 0,
+ "visited_focus": [], "failed_stages": 0, "recovery_required": False}
+ memory.execution_constraints.append(control)
+ self.control, self.content_state = control, _content_state(targets)
+ by_id = {t.id: t for t in targets}
+ for operation, candidates in space.target_heads.items():
+ for candidate in candidates:
+ target = by_id.get(candidate.target_id)
+ identity = target.semantic_key if target else candidate.metadata.get("target_label", "")
+ candidate.metadata["execution_family"] = _navigation_family(
+ candidate.operation, _keys(candidate))
+ if candidate.operation in {"click", "double_click"} and identity:
+ candidate.metadata["execution_family"] = "target:" + identity.casefold()
+ space.target_heads[operation] = [c for c in candidates if c.metadata.get("commit_action")
+ or c.metadata.get("execution_family") not in control["blocked_families"]]
+ active = targets
+ if bounds := limits.get("overlay_bounds"):
+ x, y, width, height = bounds
+ active = [t for t in targets if None not in (t.x, t.y, t.width, t.height)
+ and x <= t.x + t.width // 2 < x + width and y <= t.y + t.height // 2 < y + height]
+ elif limits:
+ ids = {c.target_id for c in space.candidates() if c.target_id}
+ active = [t for t in targets if t.id in ids]
+ active_inputs = [v for v in stage["inputs"]
+ if self.fields.get(_binding(v), {}).get("phase") != "consumed"]
+ frontier = input_frontier(space, active, active_inputs)
+ typing = next((c for c in space.candidates() if c.operation == "type_text"), None)
+ specific = next((c.description for c in space.candidates() if c.metadata.get("commit_action")),
+ typing.description + f" Supplied text: {typing.metadata['text']!r}." if typing
+ else frontier["next_intent"])
+ intent = specific or interaction.get("next_intent", "") or stage["goal"]
+ if is_dialog:
+ intent = f"Within the active {overlay!r} dialog: " + intent
+ requested = any(c.source == "goal_input" or c.metadata.get("commit_action")
+ for c in space.candidates()) or any(
+ not launcher_input(v) and matches_field(v["text"], node.get("name", ""))
+ for v in stage["inputs"] for node in root.iter())
+ recovery = bool(is_dialog and limits.get("overlay_kind") != "task_dialog" and not requested
+ and (limits.get("overlay_kind") == "obstruction" or control["recovery_required"]))
+ if recovery:
+ memory.block("input_obstructed", key, overlay, observation_ref)
+ intent = (f"Restore input access blocked by the active {overlay!r} dialog. Use its visible "
+ "dismissal controls; do not edit the background or repeat failed shortcuts.")
+ if limits and not is_dialog:
+ intent = limits["next_intent"] + (" Next legal input: " + specific if specific else "")
+ if control.get("completion_rejected"):
+ if self.rejected_states.get(key) == self.content_state:
+ space.controls.pop("done", None)
+ else:
+ control.pop("completion_rejected", None)
+ memory.resolve_blocker(key, observation_ref)
+ if frontier["unbound_inputs"] and not any(c.source == "goal_input" for c in space.candidates()):
+ memory.block("target_unbound", key, ", ".join(frontier["unbound_inputs"]), observation_ref)
+ return {**interaction, **frontier, **limits, "next_intent": intent,
+ "scope_recovery": recovery,
+ "active_blocker": deepcopy(memory.active_blocker),
+ "execution_control": deepcopy(control)}
+
+ def choice_context(self, context):
+ """One current objective, not the original planner's speculative navigation script."""
+ scoped = "blocking_overlay" in context or context.get("document_recovery") or context.get(
+ "application_recovery")
+ recovery = context.get("scope_recovery", False)
+ restoring = recovery or context.get("document_recovery") or context.get("application_recovery")
+ return {"objective": context.get("next_intent") if restoring else context["goal"],
+ "delegated_goal": context["goal"], "next_action_hint": context.get("next_intent", ""),
+ "method": "" if restoring else context.get("method", ""),
+ "guidance": "" if restoring else context.get("guidance", ""),
+ "exit_condition": ("The obstruction is gone; refresh and resume the delegated objective."
+ if recovery else context["completion_criteria"]),
+ "input_owner": context.get("blocking_overlay", context.get("target_application")),
+ "constraints": context.get("constraints", []),
+ "inputs": [] if recovery else [v for v in context["inputs"]
+ if v["label"] not in context.get("unbound_inputs", [])] if scoped
+ else [v for v in context["inputs"]
+ if self.fields.get(_binding(v), {}).get("phase") != "consumed"],
+ "unbound_inputs": [] if recovery else context.get("unbound_inputs", []),
+ "confirmed_inputs": context.get("satisfied_inputs", []),
+ "failed_paths": context.get("execution_control", {}).get("blocked_families", []),
+ "failures": [{k: f[k] for k in ("error", "count") if k in f}
+ for f in context.get("failures", [])[-2:]],
+ "active_blocker": context.get("active_blocker"),
+ "grounding_exhausted": context.get("grounding_exhausted", False),
+ "scope_rule": ("Interact only with the active input owner. A business dialog may be "
+ "the task itself; do not dismiss it just because it is a dialog."
+ if scoped else "Use the current observation, not historical UI state."),
+ "actions_remaining": context.get("actions_remaining")}
+
+ def finish(self, memory, result, history):
+ """Only a resolved execution condition releases a failed path; unrelated facts cannot."""
+ stage = memory.active_stage
+ if stage["route"] != "gui" or self.control is None:
+ return
+ control = self.control
+ if not history and result["stage_outcome"] != "met":
+ control["completion_rejected"] = True
+ self.rejected_states[control["scope_key"]] = self.content_state
+ memory.block("rejected_completion", control["scope_key"], result["summary"],
+ next(iter(result["evidence_refs"]), ""),
+ attempt={"action": "done", "result": result["stage_outcome"]})
+ elif not history and result["stage_outcome"] == "met":
+ return # A genuinely satisfied initial state need not be modified.
+ needed = {f["key"] for f in stage.get("fact_targets", [])} & memory.source_keys
+ current_refs = {h["evidence_ref"] for h in history}
+ source_complete = (stage["purpose"] == "prepare" and not control["global_scope"]
+ and result["stage_outcome"] == "met"
+ and any(f["key"] in needed and f["status"] == "confirmed"
+ and set(f["evidence_refs"]) & current_refs for f in result["facts"]))
+ resolved = result["stage_outcome"] == "met" and result.get("progress_kind") in {
+ "navigation", "content", "save", "inspection"}
+ if source_complete or result["unit_met"] or resolved:
+ control.update(attempts={}, without_effect=0, visited_focus=[], failed_stages=0,
+ ground_calls=0, ground_seconds=0.0, blocked_families=[], recovery_required=False)
+ else:
+ blocked = [family for family, count in control["attempts"].items()
+ if count >= (3 if family.startswith("target:") else 4)]
+ control["failed_stages"] += 1
+ if blocked or control["failed_stages"] >= 2:
+ control["blocked_families"] = sorted(set(control["blocked_families"] + blocked))
+ control["recovery_required"] = True
+ control["last_evidence_ref"] = (history[-1]["evidence_ref"] if history
+ else next(iter(result["evidence_refs"]), ""))
+ memory.work_items[stage["work_item_id"]]["recovery_required"] = True
+ if not control["recovery_required"]:
+ memory.work_items[stage["work_item_id"]]["recovery_required"] = False
+ self.checkpoint_due = False
+
+ def constrain(self, space, targets, stage, memory, tree=""):
+ if self.stage_id != stage["id"]:
+ self.stage_id, self.fields = stage["id"], {}
+ self.invalidate_visual = False
+ by_id = {t.id: t for t in targets}
+ application = stage.get("target_application")
+ for operation, candidates in space.target_heads.items():
+ space.target_heads[operation] = [c for c in candidates if not (
+ application and (t := by_id.get(c.target_id)) and t.application == "gnome-shell"
+ and t.role in {"push_button", "button"} and t.name.casefold() != "files"
+ and application_mismatch(application,
+ ET.tostring(ET.Element("desktop-frame", foreground_class=t.name), encoding="unicode"),
+ known_only=True))]
+ available = {}
+ for candidate in space.candidates():
+ if candidate.operation != "type_text":
+ continue
+ for value in stage["inputs"]:
+ if (candidate.metadata.get("text", "").rstrip() != value["text"].rstrip()
+ or launcher_input(value)):
+ continue
+ if not matches_field(value.get("target_hint", value["label"]),
+ candidate.metadata.get("target_hint", "")):
+ continue
+ key = _binding(value)
+ field = self.fields.setdefault(key, {
+ "target": value.get("target_hint", value["label"]),
+ "expected_value": value["text"].rstrip(),
+ "phase": "needs_input", "commit": "enter" if _command_input(value) else None})
+ target = by_id.get(candidate.target_id)
+ point = ([target.x + target.width // 2, target.y + target.height // 2] if target
+ else candidate.metadata.get("visual_point"))
+ available[key] = point
+ candidate.metadata["interaction_binding"] = key
+ if field["commit"]:
+ candidate.metadata["text"] = candidate.metadata["text"].rstrip("\r\n")
+ # Satisfied semantic fields need no typing candidate but may still need an explicit commit.
+ for value in stage["inputs"]:
+ key = _binding(value)
+ target = _input_target(value, targets)
+ field = self.fields.get(key)
+ title = ET.fromstring(tree or "").get("foreground_title", "")
+ if (field and value["text"].startswith("/") and field["phase"] in {
+ "needs_commit", "submitted_unconfirmed", "typed_unconfirmed"}
+ and Path(value["text"]).name.casefold() in title.casefold()):
+ # The opened document is evidence; a vanished path box is expected.
+ field["phase"] = "consumed"
+ if key in self.fields and target is not None:
+ available[key] = [target.x + target.width // 2, target.y + target.height // 2]
+ if input_satisfied(value, target) and not self.fields[key]["commit"]:
+ self.fields[key]["phase"] = "confirmed"
+ commits = []
+ for key, field in self.fields.items():
+ if field["phase"] == "needs_commit" and available.get(key):
+ commits.append(ActionCandidate(
+ id="COMMIT:" + key, operation="keypress", source="interaction",
+ description=f"Focus {field['target']!r} and press ENTER to submit the previously typed "
+ "command. Its result still needs observation; do not request completion yet.",
+ metadata={"key_sequence": [["enter"]], "focus_point": available[key],
+ "interaction_binding": key, "commit_action": True}))
+ for operation, candidates in space.target_heads.items():
+ space.target_heads[operation] = [c for c in candidates if not (
+ (key := c.metadata.get("interaction_binding")) in self.fields
+ and self.fields[key]["phase"] != "needs_input")]
+ if commits:
+ space.target_heads["keypress"] = [c for c in space.target_heads["keypress"]
+ if _keys(c) != [["enter"]]] + commits
+ pending = [c for c in space.candidates() if c.operation == "type_text"]
+ for operation, candidates in space.target_heads.items():
+ space.target_heads[operation] = [c for c in candidates if not (
+ c.operation == "type_text" and memory.blocked_command(
+ c.metadata.get("text", ""), application))]
+ pending = [c for c in pending if not memory.blocked_command(c.metadata.get("text", ""), application)]
+ if pending or commits:
+ space.controls.pop("done", None)
+ uncertain = [v["target"] for v in self.fields.values() if v["phase"] == "typed_unconfirmed"]
+ label = (pending[0].metadata.get("field_label") or pending[0].metadata.get("target_hint", "field")
+ if pending else "")
+ typing = (f"Enter supplied {pending[0].metadata['text']!r} into the observed "
+ f"{label!r}."
+ if pending else "")
+ if pending and pending[0].metadata.get("choice_input"):
+ typing = pending[0].description
+ intent = (commits[0].description if commits else typing if pending else
+ "Input dispatched but value unconfirmed for " + ", ".join(uncertain)
+ + ". Inspect the visible result and continue: " + stage["goal"] if uncertain else "")
+ return {"interaction": self.snapshot(), "next_intent": intent} if intent else {
+ "interaction": self.snapshot()}
+
+ def snapshot(self):
+ return {"stage_id": self.stage_id, "fields": deepcopy(list(self.fields.values())),
+ "execution_control": deepcopy(self.control), "local_effect": self.local_effect}
diff --git a/src/tandem/gui/semantics.py b/src/tandem/gui/semantics.py
index e68904d..37d67ac 100644
--- a/src/tandem/gui/semantics.py
+++ b/src/tandem/gui/semantics.py
@@ -17,7 +17,8 @@ def _norm(value: str) -> str:
def _key(target: UITarget) -> str:
- return "|".join((target.role, _norm(target.name), *(_norm(p) for p in target.parent_context)))
+ return "|".join((target.role, _norm(target.name), *(_norm(p) for p in target.parent_context),
+ _norm(target.nearby_text) if "dom" in target.source else ""))
def _bounds(element: ET.Element) -> tuple[int, int, int, int] | None:
@@ -38,11 +39,13 @@ def active(element: ET.Element) -> bool:
for key, value in element.attrib.items()
)
- for app in root:
+ apps = [root] if root.tag.rsplit("}", 1)[-1] == "application" else list(root)
+ for app in apps:
app_active = active(app) or any(active(child) for child in app)
chrome_active = chrome_active or (
app_active and "chrome" in app.attrib.get("name", "").lower()
)
+ owners = {element: app.get("name", "") for app in apps for element in app.iter()}
for element in root.iter():
role = element.tag.rsplit("}", 1)[-1].replace("-", "_")
if role not in _LABEL_ROLES:
@@ -51,7 +54,8 @@ def active(element: ET.Element) -> bool:
text = " ".join((element.attrib.get("name") or element.text or "").split())
if box and text and len(text) <= 120:
x, y, width, height = box
- labels.append({"text": text, "box": box, "cx": x + width / 2, "cy": y + height / 2})
+ labels.append({"text": text, "box": box, "cx": x + width / 2, "cy": y + height / 2,
+ "application": owners.get(element, "")})
return labels, chrome_active
@@ -61,9 +65,13 @@ def _nearest_label(target: UITarget, labels: list[dict]) -> str:
cx, cy = target.x + target.width / 2, target.y + target.height / 2
best, best_score = "", float("inf")
for item in labels:
+ if item["application"] != target.application:
+ continue
x, y, width, height = item["box"]
- left = x + width <= target.x + 4 and abs(item["cy"] - cy) <= max(14, target.height)
- above = y + height <= target.y + 4 and x + width >= target.x and x <= target.x + target.width
+ left = (0 <= target.x - x - width <= 160
+ and abs(item["cy"] - cy) <= max(14, min(40, target.height)))
+ above = (0 <= target.y - y - height <= 48 and x + width >= target.x
+ and x <= target.x + target.width)
if not left and not above:
continue
score = (
@@ -137,7 +145,7 @@ def _dom_target(item: dict, snapshot_id: str) -> UITarget | None:
showing=True,
selected=item.get("selected"),
checked=item.get("checked"),
- focused=None,
+ focused=item.get("focused"),
x=x,
y=y,
width=width,
@@ -151,6 +159,8 @@ def _dom_target(item: dict, snapshot_id: str) -> UITarget | None:
scope="web_content",
label_source=str(item.get("label_source") or "dom"),
operations=tuple(item.get("operations") or ("click",)),
+ input_type=str(item.get("input_type") or ""),
+ date_order=tuple(item.get("date_order") or ()),
)
return replace(target, semantic_key=_key(target))
@@ -164,7 +174,7 @@ def _merge(base: UITarget, dom: UITarget) -> UITarget:
merged = replace(
base,
name=dom.name if use_dom_name else base.name,
- value=dom.value or base.value,
+ value=dom.value if "type_text" in dom.operations else dom.value or base.value,
checked=dom.checked if dom.checked is not None else base.checked,
selected=dom.selected if dom.selected is not None else base.selected,
source="a11y+dom",
@@ -173,10 +183,50 @@ def _merge(base: UITarget, dom: UITarget) -> UITarget:
scope="web_content",
label_source=dom.label_source,
operations=dom.operations,
+ input_type=dom.input_type or base.input_type,
+ date_order=dom.date_order or base.date_order,
)
return replace(merged, semantic_key=_key(merged))
+def _date_fields(targets):
+ """Compose adjacent, explicitly named date segments; never infer order from locale or a screenshot."""
+ parts = [replace(t, name=t.name.split(" ", 1)[0],
+ nearby_text=t.nearby_text or t.name.partition(" ")[2])
+ for t in targets if t.role == "spin_button"
+ and t.name.casefold().split(" ", 1)[0] in {"day", "month", "year"}]
+ for day in [t for t in parts if t.name.casefold() == "day"]:
+ nearby = [t for t in parts if t.application == day.application
+ and t.parent_context == day.parent_context and abs(t.y - day.y) <= 4
+ and abs(t.x - day.x) <= 180]
+ if len(nearby) != 3 or {t.name.casefold() for t in nearby} != {"day", "month", "year"}:
+ continue
+ name = next((t.nearby_text for t in nearby if re.search(r"[\w]{3}", t.nearby_text)), "")
+ if not name:
+ continue
+ ordered = sorted(nearby, key=lambda t: t.x)
+ values = {t.name.casefold(): t.value for t in nearby}
+ value = (f"{int(values['year']):04d}-{int(values['month']):02d}-{int(values['day']):02d}"
+ if all(v.isdigit() and int(v) > 0 for v in values.values()) else "")
+ left, top = min(t.x for t in nearby), min(t.y for t in nearby)
+ field = replace(day, id="date:" + day.id, role="entry", name=name, value=value,
+ x=left, y=top, width=max(t.x + t.width for t in nearby) - left,
+ height=max(t.y + t.height for t in nearby) - top, description="Segmented date",
+ operations=("click", "type_text"), input_type="date",
+ date_order=tuple(t.name.casefold() for t in ordered))
+ yield replace(field, semantic_key=_key(field))
+
+
+def unrepresented_window(root: ET.Element | None) -> str:
+ """A foreground window absent from A11y must not inherit the background page's controls."""
+ if root is None:
+ return ""
+ title = root.get("foreground_title", "")
+ windows = {n.get("name") for n in root.iter()
+ if n.tag.rsplit("}", 1)[-1] in {"frame", "dialog"} and n.get("name")}
+ return title if windows and title not in windows else ""
+
+
def enrich_targets(
targets: list[UITarget],
accessibility_tree: str | None,
@@ -188,6 +238,8 @@ def enrich_targets(
root = ET.fromstring(accessibility_tree) if accessibility_tree else None
except ET.ParseError:
root = None
+ if unrepresented_window(root):
+ return [] # The existing visual fallback must locate this real input owner first.
labels, chrome_active = _a11y_context(root)
content_y = _content_y(targets, browser_snapshot)
enriched = []
@@ -196,7 +248,9 @@ def enrich_targets(
if target.role == "page_tab":
name = _MEMORY_USAGE.sub("", name)
nearby = target.nearby_text or _nearest_label(target, labels)
- if not name and nearby:
+ if not name and nearby and (target.role in {"entry", "text", "textbox", "searchbox",
+ "spin_button", "combo_box"}
+ or "type_text" in target.operations):
name, source = nearby, "a11y_nearby"
if not name and target.focused:
name, source = f"Focused {target.role.replace('_', ' ')}", "synthetic"
@@ -210,6 +264,7 @@ def enrich_targets(
label_source=source,
)
enriched.append(replace(item, semantic_key=_key(item)))
+ enriched.extend(list(_date_fields(enriched)))
chrome_targets = any(
"chrome" in target.application.lower() or "chromium" in target.application.lower()
for target in targets
diff --git a/src/tandem/gui/targets.py b/src/tandem/gui/targets.py
index 0f7bd8a..5e9ba24 100644
--- a/src/tandem/gui/targets.py
+++ b/src/tandem/gui/targets.py
@@ -33,6 +33,10 @@ class UITarget:
scope: str = "native"
label_source: str = "a11y"
operations: tuple[str, ...] = ()
+ range_min: float | None = None
+ range_max: float | None = None
+ input_type: str = ""
+ date_order: tuple[str, ...] = ()
def compact(self) -> dict:
return {key: value for key, value in asdict(self).items() if value not in (None, "", (), False)}
@@ -100,6 +104,32 @@ def _attribute(element: ET.Element, name: str) -> str:
return ""
+def _number(element, name):
+ value = _attribute(element, name)
+ return float(value) if re.fullmatch(r"-?\d+(?:\.\d+)?", value) else None
+
+
+def _is_overlay(element):
+ return _local(element.tag) in {"dialog", "alert", "popup_menu"} or (
+ _local(element.tag) == "menu" and _attribute(element, "ispopup") in {"true", "auto", "menu"})
+
+
+def _field_labels(root):
+ labels = {}
+ for parent in root.iter():
+ preceding = ""
+ for child in parent:
+ if _local(child.tag) in {"label", "static"}:
+ preceding = str(child.get("name") or child.text or "").strip()
+ continue
+ fields = [e for e in child.iter() if _state(e, "editable") is True
+ or _local(e.tag) in {"entry", "textbox", "searchbox", "combo_box", "spin_button"}]
+ if preceding and len(fields) == 1:
+ labels[fields[0]] = preceding
+ preceding = ""
+ return labels
+
+
class A11yTargetBuilder:
def build(
self,
@@ -108,23 +138,43 @@ def build(
max_targets: int = 64,
infer_button_labels: bool = False,
include_sliders: bool = False,
+ context: str = "",
+ filter_occluded: bool = False,
) -> list[UITarget]:
root = ET.fromstring(raw_tree)
targets: list[UITarget] = []
+ field_labels = _field_labels(root) if filter_occluded else {}
+ target_nodes, overlays = {}, {}
+ if filter_occluded:
+ for node in root.iter():
+ if (_is_overlay(node) and _state(node, "showing") is True
+ and _state(node, "visible") is not False):
+ position, size = _pair(_attribute(node, "screencoord")), _pair(_attribute(node, "size"))
+ if position and size and min(size) > 0:
+ overlays.setdefault((*position, *size), set()).update(node.iter())
- def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name: str) -> None:
+ def visit(element, active, parents, app_name, modal=False, roles=(), file_manager=False):
active = active or _state(element, "active") is True
role = _local(element.tag)
+ modal = modal or _is_overlay(element)
raw_name = str(element.attrib.get("name") or "").strip()
name = raw_name
if role == "application":
app_name = name
label_source = "a11y" if name else "unresolved"
+ if not name and element in field_labels:
+ name, label_source = field_labels[element], "a11y_adjacent"
+ if (not name and filter_occluded and "gnome-shell" in parents
+ and "Overview" in parents and _state(element, "editable") is True):
+ name, label_source = "Desktop search", "a11y_context"
+ file_manager = file_manager or "nautilus" in app_name.lower() or "file chooser" in name.lower()
value = (
str(element.text or "").strip()
if role in {"text", "textbox", "entry", "searchbox", "spin_button"}
- else ""
+ else _attribute(element, "value") if role == "slider" else ""
)
+ if role == "spin_button" and not value and (number := _number(element, "value")) is not None:
+ value = format(number, "g")
position = _pair(_attribute(element, "screencoord"))
size = _pair(_attribute(element, "size"))
if infer_button_labels and not name and role in {"button", "push_button"} and position and size:
@@ -149,6 +199,9 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name:
and active
and (
any("file chooser" in item.lower() or item.lower() == "places" for item in parents)
+ or filter_occluded and file_manager and any(r in {
+ "list_item", "tree_item", "button", "push_button", "menu_item", "option", "link"}
+ for r in roles)
or bool(
re.search(
r"\b(?:monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b", name, re.I
@@ -158,7 +211,8 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name:
)
if (
(active or system_target)
- and (role in SUPPORTED_ROLES or safe_label or (include_sliders and role == "slider"))
+ and (role in SUPPORTED_ROLES or safe_label or (include_sliders and role == "slider")
+ or filter_occluded and (role == "image" or file_manager and role == "canvas"))
and (name or role in SUPPORTED_ROLES)
and interactive_text
and enabled is not False
@@ -168,7 +222,6 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name:
and size
and size[0] > 0
and size[1] > 0
- and len(targets) < max_targets
):
normalized_name = " ".join(name.lower().split())
parent_context = tuple(item for item in parents[-3:] if item)
@@ -198,10 +251,24 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name:
or _attribute(element, "placeholder-text")
),
label_source=label_source,
+ source="a11y_dialog" if filter_occluded and modal else "a11y",
+ range_min=_number(element, "min") if role == "slider" else None,
+ range_max=_number(element, "max") if role == "slider" else None,
)
)
+ target_nodes[targets[-1].id] = element
for child in element:
- visit(child, active, context[-4:], app_name)
+ visit(child, active, context[-4:], app_name, modal, (*roles, role), file_manager)
visit(root, False, (), "")
- return targets
+ if filter_occluded:
+ targets = [t for t in targets if not any(
+ target_nodes[t.id] not in nodes and x <= t.x + t.width // 2 < x + width
+ and y <= t.y + t.height // 2 < y + height
+ for (x, y, width, height), nodes in overlays.items())]
+ if context:
+ words = set(re.findall(r"\w+", context.lower()))
+ targets.sort(key=lambda t: (t.source == "a11y_dialog",
+ len(words & set(re.findall(r"\w+", t.name.lower()))), t.focused is True,
+ t.role in {"button", "push_button", "combo_box", "entry", "textbox"}), reverse=True)
+ return targets[:max_targets]
diff --git a/src/tandem/memory/__init__.py b/src/tandem/memory/__init__.py
new file mode 100644
index 0000000..e6c80da
--- /dev/null
+++ b/src/tandem/memory/__init__.py
@@ -0,0 +1 @@
+"""Task-scoped source facts, delivery progress and execution evidence."""
diff --git a/src/tandem/memory/context.py b/src/tandem/memory/context.py
new file mode 100644
index 0000000..33d304f
--- /dev/null
+++ b/src/tandem/memory/context.py
@@ -0,0 +1,83 @@
+"""Compact role-specific views. Raw receipts/screenshots remain in the episode record."""
+
+import re
+
+
+def _fact_view(fact, limit=2400):
+ view = {k: v for k, v in fact.items() if k not in {"observations", "entries"}}
+ if len(view["value"]) > limit:
+ view["value"] = (view["value"][:limit // 2]
+ + "\n[excerpt; request read_memory for the stored content]\n"
+ + view["value"][-limit // 2:])
+ view["value_excerpt_only"] = True
+ return view
+
+
+def planning_context(memory):
+ ready = memory.ready_items()[:4]
+ keys = {f["key"] for item in ready for f in (
+ memory.preparation_target(item)[0]["facts_needed"] if item["kind"] == "prepare"
+ else item["facts_needed"])}
+ facts = [f for f in memory.facts.values() if f["scope"] == "task"]
+ auxiliary = {f["key"] for f in facts[-8:]}
+ visible = [_fact_view(f) for f in facts if f["key"] in keys | auxiliary]
+ return {"requirements": [{k: r[k] for k in ("id", "requirement", "kind", "status")}
+ for r in memory.requirements],
+ "ready_work_items": [{**item, "can_start_partial": item["kind"] == "deliver",
+ "next_acquisition": {
+ "unit": memory.preparation_target(item)[0]["goal"],
+ "missing_facts": memory.preparation_target(item)[1]}}
+ for item in ready],
+ "completed_work": [i["id"] for i in memory.work_items.values() if i["status"] == "done"],
+ "known_facts": visible,
+ "fact_index": [{**{k: f[k] for k in ("key", "status", "source_ref")},
+ "complete": f.get("complete", True)} for f in facts],
+ "historical_ui_notes": [f for f in memory.facts.values() if f["scope"] == "stage"][-6:],
+ "materials": [{"source_ref": m["source_ref"], "covered_parts": m["covered_parts"],
+ "summary_excerpt": m["summary"][:1800],
+ "summary_excerpt_only": len(m["summary"]) > 1800}
+ for m in list(memory.materials.values())[-8:]],
+ "artifacts": list(memory.artifacts.values()), "failures": memory.failures[-12:],
+ "pending_publications": [a for a in memory.artifacts.values() if a.get("delivery_pending")],
+ "unsupported_methods": [{**f["observed_method"], "observed_error": f["error"],
+ "evidence_refs": f["evidence_refs"]}
+ for f in memory.failures if f.get("observed_method")],
+ "observed_missing_modules": sorted({name for f in memory.failures for name in
+ re.findall(r"No module named ['\"]([^'\"]+)['\"]", f["error"])}),
+ "execution_constraints": memory.execution_constraints[-12:],
+ "active_blocker": memory.active_blocker,
+ "memory_update_rejections": memory.last_rejections,
+ "last_interaction_historical": memory.last_interaction,
+ "stages_without_progress": memory.stats["stages_without_progress"]}
+
+
+def execution_context(memory):
+ stage = memory.active_stage
+ item = memory.work_items[stage["work_item_id"]]
+ return {"goal": stage["goal"], "completion_criteria": stage["completion_criteria"],
+ "method": stage["method"],
+ "constraints": [r["requirement"] for r in memory.requirements if r.get("kind") == "constraint"],
+ "inputs": stage["inputs"], "guidance": stage["guidance"],
+ "target_application": stage.get("target_application"),
+ "facts": [_fact_view(memory.facts[k]) for k in stage["input_fact_keys"] if k in memory.facts],
+ "delivery_state": item["delivery_state"], "remaining": item["remaining"],
+ "completed_milestones": item["milestones"],
+ "failures": [f for f in memory.failures if f["work_item_id"] == item["id"]][-4:],
+ "execution_constraints": [c for c in memory.execution_constraints
+ if c["work_item_id"] == item["id"] or c.get("global_scope")][-6:],
+ "active_blocker": memory.active_blocker,
+ "actions_remaining": stage["max_actions"] - stage["actions"],
+ "control": "Choose each next action from the NEW observation. DONE requests stage verification, "
+ "not whole-task completion. Do not repeat satisfied input or toggle states."}
+
+
+def verification_context(memory):
+ """Independent stage evidence remains available at whole-task verification."""
+ return {"requirements": memory.requirements,
+ "work_items": [{k: item[k] for k in ("id", "goal", "status", "delivery_state", "remaining",
+ "evidence_refs", "milestones")}
+ for item in memory.work_items.values() if item["kind"] == "deliver"],
+ "source_facts": [_fact_view(f) for f in memory.facts.values()
+ if f["scope"] == "task" and f["status"] == "confirmed"],
+ "artifacts": list(memory.artifacts.values()),
+ "active_blocker": memory.active_blocker}
diff --git a/src/tandem/memory/state.py b/src/tandem/memory/state.py
new file mode 100644
index 0000000..b66d1af
--- /dev/null
+++ b/src/tandem/memory/state.py
@@ -0,0 +1,355 @@
+"""Host-owned task memory. Plans request work; stage verification commits progress."""
+
+import json
+import re
+from copy import deepcopy
+from pathlib import Path
+
+LEVELS = {"none": 0, "implemented": 1, "saved": 2, "verified": 3}
+
+
+class TaskMemory:
+ def __init__(self, instruction, requirements):
+ self.instruction = instruction
+ self.requirements = deepcopy(requirements)
+ ids = [r["id"].strip() for r in self.requirements]
+ if not all(ids) or len(set(ids)) != len(ids):
+ raise ValueError("Requirement IDs must be nonempty and unique")
+ self.facts = {}
+ self.materials = {}
+ self.work_items = {}
+ self.artifacts = {}
+ self.failures = []
+ self.execution_constraints = []
+ self.active_stage = None
+ self.stage_count = 0
+ self.revision = 0
+ self.known_evidence = set()
+ self.last_rejections = []
+ self.last_interaction = {}
+ self.active_blocker = None
+ sources = {f["key"]: f for r in self.requirements for f in r["facts_needed"]}
+ self.source_keys = set(sources)
+ self.stats = {"stages": 0, "verified_stages": 0, "gui_actions": 0,
+ "cli_actions": 0, "new_facts": 0, "delivery_advances": 0,
+ "source_parts": 0, "milestone_advances": 0, "stages_without_progress": 0}
+ for req in self.requirements:
+ req.update(status="unknown", evidence_refs=[])
+ if req.get("kind") == "constraint":
+ continue
+ needed = [sources[key] for key in dict.fromkeys(
+ key for keys in req["unit_fact_keys"] for key in keys)]
+ preparation = f"{req['id']}:prepare"
+ if needed:
+ self._add(preparation, req, "prepare",
+ "Acquire missing source facts for a ready unit", [], needed)
+ for index, unit in enumerate(req["units"], 1):
+ keys = req["unit_fact_keys"][index - 1]
+ self._add(f"{req['id']}:unit:{index}", req, "deliver", unit, [],
+ [sources[key] for key in keys])
+
+ def _add(self, key, req, kind, goal, dependencies, facts):
+ if key in self.work_items:
+ raise ValueError("Duplicate work item: " + key)
+ self.work_items[key] = {
+ "id": key, "requirement_ids": [req["id"]], "kind": kind, "goal": goal,
+ "completion_criteria": goal, "depends_on": dependencies,
+ "facts_needed": facts, "status": "pending",
+ "delivery_state": "none", "remaining": [], "evidence_refs": [],
+ "milestones": {}, "failed_cli_stages": 0,
+ }
+
+ def _facts_ready(self, item):
+ return all(self.facts.get(f["key"], {}).get("status") == "confirmed"
+ and self.facts[f["key"]]["scope"] == "task"
+ and self.facts[f["key"]].get("complete", True) for f in item["facts_needed"])
+
+ def preparation_target(self, item):
+ units = [item] if item["kind"] == "deliver" else [
+ w for w in self.work_items.values() if w["kind"] == "deliver"
+ and w["status"] != "done" and w["requirement_ids"] == item["requirement_ids"]]
+ choices = [(unit, [f for f in unit["facts_needed"]
+ if not self._facts_ready({"facts_needed": [f]})]) for unit in units]
+ choices = [(unit, missing) for unit, missing in choices if missing]
+ return min(choices, key=lambda pair: len(pair[1])) if choices else (item, [])
+
+ def ready_items(self):
+ for item in self.work_items.values():
+ if item["kind"] == "prepare" and item["status"] != "done" and self._facts_ready(item):
+ item.update(status="done", evidence_refs=[ref for f in item["facts_needed"]
+ for ref in self.facts[f["key"]]["evidence_refs"]])
+ return sorted([i for i in self.work_items.values() if i["status"] != "done"
+ and all(self.work_items[d]["status"] == "done" for d in i["depends_on"])],
+ key=lambda i: (i["kind"] != "deliver", i.get("stalled_stages", 0),
+ len(self.preparation_target(i)[1])))
+
+ def start(self, plan, step):
+ item = self.work_items[plan["work_item_id"]]
+ if item not in self.ready_items():
+ raise ValueError("Stage work item is not ready or has already completed")
+ self.stage_count += 1
+ self.active_stage = {**deepcopy(plan), "id": f"S{self.stage_count}",
+ "started_step": step, "actions": 0, "status": "running"}
+ if plan["purpose"] == "prepare":
+ unit, missing = self.preparation_target(item)
+ self.active_stage.update(prepares_unit=unit["id"], fact_targets=deepcopy(missing))
+ item["status"] = "active"
+ self.stats["stages"] += 1
+ return self.active_stage
+
+ def record_action(self, record):
+ self.active_stage["actions"] += 1
+ self.stats["cli_actions" if record["kind"] == "cli" else "gui_actions"] += 1
+ if record["returncode"] != 0:
+ error = (record.get("stderr") or record.get("stdout") or "Execution failed")[-700:]
+ self._failure(error, [record["evidence_ref"]])
+
+ def _failure(self, error, refs, **details):
+ stage = self.active_stage
+ matches = [f for f in self.failures if f["work_item_id"] == stage["work_item_id"]
+ and f["error"] == error]
+ if matches:
+ matches[-1]["count"] += 1
+ matches[-1]["evidence_refs"] = refs
+ matches[-1].update(details)
+ else:
+ self.failures.append({"stage_id": stage["id"], "work_item_id": stage["work_item_id"],
+ "method": stage["method"], "error": error,
+ "count": 1, "evidence_refs": refs, **details})
+
+ def blocked_command(self, text, application=None):
+ """Only an observed unsupported command is blocked; advice is never a capability fact."""
+ command = re.match(r"\s*([A-Za-z]\w*)\s*\(", text)
+ if not command:
+ return ""
+ for failure in self.failures:
+ method = failure.get("observed_method", {})
+ if method.get("command_name", "").casefold() != command[1].casefold():
+ continue
+ known = method.get("application", "").casefold()
+ words = [w for w in re.findall(r"[a-z]+", (application or "").casefold())
+ if len(w) >= 3 and w not in {"classic", "presentation", "office", "desktop"}]
+ if application and known and not any(w in known for w in words):
+ continue
+ return f"Observed unsupported command {command[1]!r}: {failure['error']}. Use another GUI method."
+ return ""
+
+ def block(self, kind, owner, subject, evidence_ref, *, attempt=None):
+ """Execution owns blockers; renaming a plan or switching units cannot clear one."""
+ previous = self.active_blocker
+ if not previous or previous["owner"] != owner or previous["status"] == "resolved":
+ self.active_blocker = {"kind": kind, "owner": owner, "subject": subject,
+ "status": "blocked", "resume_stage_id": self.active_stage["id"],
+ "work_item_id": self.active_stage["work_item_id"], "evidence_refs": [],
+ "attempts": [], "resolution_evidence_refs": []}
+ blocker = self.active_blocker
+ if kind == "input_obstructed" or blocker["kind"] != "input_obstructed":
+ blocker.update(kind=kind, subject=subject)
+ if evidence_ref and evidence_ref not in blocker["evidence_refs"]:
+ blocker["evidence_refs"].append(evidence_ref)
+ blocker["evidence_refs"] = blocker["evidence_refs"][-4:]
+ if attempt:
+ blocker["attempts"].append({**attempt, "evidence_ref": evidence_ref})
+ blocker["attempts"] = blocker["attempts"][-8:]
+ return blocker
+
+ def resolve_blocker(self, owner, evidence_ref):
+ """Require a new observed resolution, never a planner's promise of recovery."""
+ blocker = self.active_blocker
+ if blocker and blocker["owner"] == owner and evidence_ref:
+ blocker.update(status="resolved", resolution_evidence_refs=[evidence_ref])
+
+ def apply(self, result, evidence):
+ """Only the fixed active unit is writable; references must belong to real evidence."""
+ stage = self.active_stage
+ item = self.work_items[stage["work_item_id"]]
+ current = set(evidence)
+ explicit_outputs = {r.get("output_path") for r in self.requirements}
+ unpublished = [a for a in result["artifacts"] if a["target_path"] != a["working_path"]
+ or Path(a["target_path"]).is_relative_to("/tmp")
+ and a["target_path"] not in explicit_outputs]
+ pending_copy = any(a.get("delivery_pending") and a.get("work_item_id") == item["id"]
+ for a in self.artifacts.values())
+ if (stage["purpose"] != "prepare" and result["delivery_state"] in {"saved", "verified"}
+ and (unpublished and len(unpublished) == len(result["artifacts"])
+ or pending_copy and not result["artifacts"])):
+ remaining = "Publish the working copy to the user's deliverable and reopen that final file."
+ result.update(delivery_state="implemented", unit_met=False,
+ remaining=list(dict.fromkeys(result["remaining"] + [remaining])))
+ if "unit_assessment" in result:
+ result["unit_assessment"].update(status="not_met", remaining=result["remaining"])
+ for artifact in unpublished:
+ artifact["delivery_pending"] = True
+ self.known_evidence.update(current)
+ allowed = self.known_evidence
+ self.last_rejections = []
+ advanced = False
+ for fact in result["facts"]:
+ fact = deepcopy(fact)
+ refs = fact["evidence_refs"]
+ if not refs or not set(refs) <= allowed:
+ self.last_rejections.append({"key": fact["key"], "reason": "unregistered_evidence"})
+ continue
+ if fact["key"] in self.source_keys and fact["scope"] != "task":
+ self.last_rejections.append({"key": fact["key"], "reason": "source_key_requires_source_fact"})
+ continue
+ old = self.facts.get(fact["key"])
+ if (fact["key"] in self.source_keys and stage["purpose"] != "prepare"
+ and (old is None or old["value"] != fact["value"])):
+ self.last_rejections.append({"key": fact["key"],
+ "reason": "source_update_requires_acquisition_stage"})
+ continue
+ entries = fact.get("entries", {})
+ old_entries = old.get("entries", {}) if old else {}
+ conflicts = any(key in old_entries and old_entries[key] != value
+ for key, value in entries.items())
+ extending = bool(old_entries and entries and not conflicts)
+ correction = bool(fact.get("correction_reason") and set(refs) & current)
+ # Incomplete prose sources may arrive a section at a time without structured entries.
+ # Append evidence, never replace the previously observed section or unlock a partial fact.
+ if (old and old["status"] == fact["status"] == "confirmed"
+ and old["scope"] == fact["scope"] == "task" and not old.get("complete", True)
+ and not entries and not old_entries and set(refs) & current and not correction):
+ sections = deepcopy(old.get("observations", [{k: old[k] for k in
+ ("value", "source_ref", "evidence_refs")}]))
+ if fact["value"] not in [section["value"] for section in sections]:
+ sections.append({k: fact[k] for k in ("value", "source_ref", "evidence_refs")})
+ fact["observations"] = sections
+ fact["value"] = "\n".join(section["value"] for section in sections)
+ fact["evidence_refs"] = list(dict.fromkeys(old["evidence_refs"] + refs))
+ extending = True
+ if entries:
+ fact["entries"] = {**old_entries, **entries}
+ fact["value"] = json.dumps(fact["entries"], ensure_ascii=False)
+ fact["evidence_refs"] = list(dict.fromkeys((old or {}).get("evidence_refs", []) + refs))
+ fact["complete"] = bool(fact.get("complete", False) or (old or {}).get("complete", False))
+ if (old and old["status"] == "confirmed" and old["scope"] == "task"
+ and (fact["status"] != "confirmed" or fact["scope"] != "task"
+ or old.get("complete", True) and not fact.get("complete", True)
+ or old["value"] != fact["value"] and not (correction or extending)
+ or conflicts and not correction)):
+ self.last_rejections.append({"key": fact["key"], "reason": "confirmed_source_preserved"})
+ continue
+ if (fact["status"] == "confirmed" and fact["scope"] == "task"
+ and (old is None or old["value"] != fact["value"] or old["status"] != fact["status"]
+ or old.get("complete", True) != fact.get("complete", True))):
+ advanced = advanced or fact["key"] in self.source_keys
+ self.stats["new_facts"] += 1
+ self.facts[fact["key"]] = {**fact, "requirement_ids": item["requirement_ids"],
+ "observed_stage": stage["id"]}
+ for material in result["materials"]:
+ if material["evidence_refs"] and set(material["evidence_refs"]) <= allowed:
+ old = self.materials.get(material["source_ref"], {})
+ new_parts = set(material["covered_parts"]) - set(old.get("covered_parts", []))
+ if new_parts:
+ self.stats["source_parts"] += len(new_parts)
+ summary = old.get("summary", "")
+ if material["summary"] not in summary:
+ summary = (summary + "\n" + material["summary"]).strip()
+ self.materials[material["source_ref"]] = {
+ **material, "summary": summary,
+ "covered_parts": list(dict.fromkeys(old.get("covered_parts", [])
+ + material["covered_parts"])),
+ "evidence_refs": list(dict.fromkeys(old.get("evidence_refs", [])
+ + material["evidence_refs"]))}
+ refs = result["evidence_refs"]
+ grounded = bool(set(refs) & current) and set(refs) <= allowed
+ level = result["delivery_state"]
+ if result.get("progress_kind") in {"navigation", "source", "none"}:
+ level = "none"
+ delivering = item["kind"] == "deliver" and stage["purpose"] != "prepare"
+ if grounded and delivering and level != "none" and result["stage_outcome"] == "met":
+ for milestone in result.get("milestones", []):
+ milestone_refs = milestone["evidence_refs"]
+ old = item["milestones"].get(milestone["key"])
+ if (set(milestone_refs) & current and set(milestone_refs) <= allowed
+ and (old is None or LEVELS[level] > LEVELS[old["delivery_state"]])):
+ item["milestones"][milestone["key"]] = {**milestone, "delivery_state": level}
+ advanced = True
+ self.stats["milestone_advances"] += 1
+ # A verified section is not a verified whole document/application.
+ if level == "verified" and not result["unit_met"]:
+ saved = any(a["save_state"] == "saved" for a in result["artifacts"])
+ level = "saved" if saved or stage["purpose"] == "save" else "implemented"
+ if (grounded and delivering and LEVELS[level] > LEVELS[item["delivery_state"]]
+ and (level != "verified" or result["stage_outcome"] == "met")):
+ item["delivery_state"] = level
+ advanced = True
+ self.stats["delivery_advances"] += 1
+ unit_check = result.get("unit_assessment", {})
+ remaining = unit_check.get("remaining", result["remaining"]) if unit_check.get(
+ "work_item_id") == item["id"] else result["remaining"]
+ item.update(status="pending", remaining=remaining, evidence_refs=refs if grounded else [])
+ preparation_met = item["kind"] == "prepare" and self._facts_ready(item)
+ if grounded and result["stage_outcome"] == "met":
+ self.stats["verified_stages"] += 1
+ if preparation_met or (delivering and result["unit_met"]
+ and level == "verified"):
+ item["status"] = "done"
+ advanced = True
+ for check in result["requirement_checks"]:
+ if check["evidence_refs"] and set(check["evidence_refs"]) <= allowed:
+ req = next((r for r in self.requirements if r["id"] == check["id"]), None)
+ if req is not None:
+ req.update(status=check["status"], evidence_refs=check["evidence_refs"])
+ for artifact in result["artifacts"]:
+ if not artifact["evidence_refs"] or not set(artifact["evidence_refs"]) <= allowed:
+ continue
+ old = self.artifacts.get(artifact["target_path"], {})
+ if (old and artifact["working_path"] != old["working_path"]
+ and (not set(artifact["evidence_refs"]) & current
+ or artifact.get("replaces_working_path") != old["working_path"])):
+ self.last_rejections.append({"key": artifact["target_path"],
+ "reason": "working_copy_change_requires_explicit_migration"})
+ continue
+ self.artifacts[artifact["target_path"]] = {
+ **artifact, "work_item_id": item["id"], "revision": old.get("revision", 0) + 1}
+ if artifact["save_state"] == "saved" and artifact["target_path"] == artifact["working_path"]:
+ for previous in self.artifacts.values():
+ if previous["working_path"] == artifact.get("replaces_working_path"):
+ previous.pop("delivery_pending", None)
+ for key in result["invalidated_work_item_ids"]:
+ if key in self.work_items and self.work_items[key]["kind"] == "deliver" and grounded:
+ self.work_items[key].update(status="pending", delivery_state="none",
+ milestones={},
+ remaining=["Contradictory evidence requires reinspection"])
+ for req in self.requirements:
+ if req["id"] in self.work_items[key]["requirement_ids"]:
+ req.update(status="uncertain", evidence_refs=list(current))
+ for error in result.get("observed_errors", []):
+ error_refs = set(error["evidence_refs"])
+ if not error_refs or not error_refs <= current:
+ continue
+ details = {"suggested_actions": result["remaining"]}
+ name = error.get("command_name", "")
+ if (name and re.fullmatch(r"[A-Za-z]\w*", name)
+ and re.search(r"unknown command\s*:\s*" + re.escape(name) + r"\b",
+ error["message"], re.I)):
+ details["observed_method"] = {"application": error.get("application", ""),
+ "command_name": name}
+ self._failure(error["message"], error["evidence_refs"], **details)
+ if result["stage_outcome"] != "met" and grounded:
+ self._failure(result["summary"], refs, suggested_actions=result["remaining"])
+ self.stats["stages_without_progress"] = 0 if advanced else self.stats["stages_without_progress"] + 1
+ item["stalled_stages"] = 0 if advanced else item.get("stalled_stages", 0) + 1
+ item["last_attempt"] = {k: stage[k] for k in ("route", "purpose", "method", "command", "inputs")}
+ item["last_attempt"]["failure_kind"] = result.get("failure_kind", "")
+ failed_cli = stage["route"] == "cli" and result.get("failure_kind") == "execution_error"
+ item["failed_cli_stages"] = item["failed_cli_stages"] + 1 if failed_cli else 0
+ self.revision += 1
+ self.last_interaction = deepcopy(stage.get("interaction", {}))
+ self.active_stage = None
+ self.ready_items()
+ return advanced
+
+ def snapshot(self):
+ return deepcopy({"schema_version": 2, "revision": self.revision, "instruction": self.instruction,
+ "requirements": self.requirements, "materials": list(self.materials.values()),
+ "facts": list(self.facts.values()), "work_items": list(self.work_items.values()),
+ "artifacts": list(self.artifacts.values()), "failures": self.failures,
+ "execution_constraints": self.execution_constraints,
+ "memory_update_rejections": self.last_rejections,
+ "last_interaction": self.last_interaction,
+ "active_blocker": self.active_blocker,
+ "active_stage": self.active_stage, "stats": self.stats,
+ "evidence_refs": sorted(self.known_evidence)})
diff --git a/src/tandem/models/jev.py b/src/tandem/models/jev.py
index 18950c9..342b2b8 100644
--- a/src/tandem/models/jev.py
+++ b/src/tandem/models/jev.py
@@ -21,11 +21,12 @@
class JevOmniPolicy:
def __init__(self, endpoint: str, timeout_seconds: float = 300, opener=None,
- image_stage: tuple[str, str] | None = None, endpoint_provider=None):
+ image_stage: tuple[str, str] | None = None, endpoint_provider=None, joint_actions=False):
self.endpoint = str(endpoint).rstrip("/")
self.timeout_seconds = timeout_seconds
self.opener = opener or urllib.request.build_opener(urllib.request.ProxyHandler({}))
self.endpoint_provider = endpoint_provider
+ self.joint_actions = joint_actions
# (写视角目录, 排序服务宿主机读视角目录):/rank 要求 image_path 在服务端可读,
# 截图本体在 adpc 时先暂存到 evobox 中转目录(按内容哈希命名去重)。
self.image_stage = image_stage
@@ -164,6 +165,18 @@ def predict(choice_request: dict[str, Any]) -> tuple[list[float], bool]:
candidate for candidate in candidates if candidate.operation == selected_operation
]
action_request = action_requests[selected_operation]
+ if self.joint_actions:
+ # Keep the operation ranking diagnostic; do not prune other concrete goal actions.
+ descriptions = {choice_id: option for request in action_requests.values()
+ for choice_id, option in zip(
+ request["option_ids"], request["options"], strict=True)}
+ selected_candidates = candidates
+ action_request = {
+ "state": operation_request["state"],
+ "question": "Which complete concrete action should be performed next across ALL operations?",
+ "option_ids": [c.id for c in candidates],
+ "options": [f"{c.operation}: {descriptions[c.id]}" for c in candidates],
+ }
if len(selected_candidates) == 1:
action_scores, action_used_image = [1.0], operation_used_image
else:
diff --git a/src/tandem/models/planner_choice.py b/src/tandem/models/planner_choice.py
new file mode 100644
index 0000000..b6ffc21
--- /dev/null
+++ b/src/tandem/models/planner_choice.py
@@ -0,0 +1,56 @@
+"""Choose one offered GUI action through the planner's multimodal API and key pool."""
+
+import base64
+import json
+import time
+
+from .ranking import Ranking, build_choice_requests
+
+
+class PlannerChoicePolicy:
+ def __init__(self, client):
+ self.client = client
+
+ def rank(self, *, stage_goal, intent, active_window, targets, action_space,
+ recent_actions, screenshot=None):
+ candidates = action_space.candidates()
+ operation, requests = build_choice_requests(
+ stage_goal=stage_goal, intent=intent, active_window=active_window, targets=targets,
+ action_space=action_space, recent_actions=recent_actions)
+ descriptions = {key: text for request in requests.values()
+ for key, text in zip(request["option_ids"], request["options"], strict=True)}
+ request = {"state": operation["state"],
+ "question": "Which complete concrete action should be performed next "
+ "across ALL operations?",
+ "option_ids": [c.id for c in candidates],
+ "options": [f"{c.operation}: {descriptions[c.id]}" for c in candidates]}
+ content = [{"type": "text", "text": json.dumps(request, ensure_ascii=False)}]
+ if screenshot is not None:
+ content.append({"type": "image_url", "image_url": {"url": "data:image/png;base64,"
+ + base64.b64encode(screenshot.read_bytes()).decode("ascii")}})
+ messages = [{"role": "system", "content":
+ "Choose exactly ONE offered action for the current local objective and screenshot. "
+ "Honor the delegated goal and preservation conditions before next_action_hint; "
+ "a field hint is not permission to discard an unsaved form or skip a navigation prerequisite. "
+ "The exit condition is a desired state, NOT an observed fact. Use current evidence; "
+ "do not repeat rejected completion or ineffective paths. Return only "
+ '{"choice_id":"exact offered option_id"}. Do not invent actions, coordinates or text.'},
+ {"role": "user", "content": content}]
+ by_id = {c.id: c for c in candidates}
+
+ def validate(value):
+ okay = isinstance(value.get("choice_id"), str) and value["choice_id"] in by_id
+ if not okay:
+ messages.append({"role": "user", "content":
+ "Invalid choice_id. Return one exact offered ID; no action has been executed."})
+ return okay
+
+ started = time.perf_counter()
+ value = self.client.json_request(messages, validate, role="gui_choice")
+ winner = by_id[value["choice_id"]]
+ return Ranking(winner=winner,
+ action_ranked=[{"id": winner.id, "operation": winner.operation}],
+ operation_ranked=[], operation_margin=0.0, target_margin=0.0,
+ latency_ms=round((time.perf_counter() - started) * 1000, 2),
+ used_image=screenshot is not None, action_request=request,
+ score_type="api_choice_id")
diff --git a/src/tandem/models/ranking.py b/src/tandem/models/ranking.py
index 4bb17eb..89ee8ca 100644
--- a/src/tandem/models/ranking.py
+++ b/src/tandem/models/ranking.py
@@ -22,6 +22,7 @@ class Ranking:
confident: bool = True
operation_request: dict[str, Any] | None = None
action_request: dict[str, Any] | None = None
+ score_type: str = "fast_model_choice_softmax"
def build_choice_requests(
@@ -121,7 +122,7 @@ def ranking_dict(value: Ranking) -> dict[str, Any]:
"target_margin": value.target_margin,
"latency_ms": value.latency_ms,
"used_image": value.used_image,
- "score_type": "fast_model_choice_softmax",
+ "score_type": value.score_type,
"score_is_calibrated_correctness_probability": False,
"operation_request": value.operation_request,
"action_request": value.action_request,
diff --git a/src/tandem/models/slow.py b/src/tandem/models/slow.py
index 35dcaf9..504bdf8 100644
--- a/src/tandem/models/slow.py
+++ b/src/tandem/models/slow.py
@@ -5,18 +5,86 @@
from __future__ import annotations
+import base64
+import io
import json
import time
import urllib.error
import urllib.request
from pathlib import Path
+from PIL import Image
+
from .key_pool import ApiKeyPool, PoolExhausted, classify_api_error, retry_after_seconds
# 空正文(全留在思考通道)或截断时,下一次重试升档思考强度,避免原样重发重复失败
_REASON_EFFORT_LADDER = {"low": "medium", "medium": "high"}
+class TaskBudgetExceeded(TimeoutError):
+ """The agent's task budget ended, not an external API timeout."""
+
+
+class ModelResponseError(ValueError):
+ """Bounded repair could not produce a valid structured model response."""
+
+
+def _pack_images(payload, quality, max_bytes):
+ """Compress send copies; only exactly identical decoded frames share an image.
+
+ Mutate the existing messages list so validators can still append correction turns on retry.
+ Never write source files or discard distinct evidence frames to meet the body budget.
+ """
+ frames, pixels = [], []
+ stats = {"image_count": 0, "duplicate_images": 0, "source_image_bytes": 0}
+ for message in payload["messages"]:
+ if not isinstance(message.get("content"), list):
+ continue
+ for part in message["content"]:
+ if part.get("type") != "image_url":
+ continue
+ url = part["image_url"]["url"]
+ if not url.startswith("data:image/"):
+ continue
+ raw = base64.b64decode(url.split(",", 1)[1])
+ with Image.open(io.BytesIO(raw)) as source:
+ frame = source.convert("RGB")
+ identity = (frame.size, frame.tobytes())
+ stats["image_count"] += 1
+ stats["source_image_bytes"] += len(raw)
+ if identity in pixels:
+ ordinal = pixels.index(identity) + 1
+ part.clear()
+ part.update(type="text", text=f"Same screenshot as supplied image #{ordinal} (exact pixels).")
+ stats["duplicate_images"] += 1
+ else:
+ pixels.append(identity)
+ frames.append((part, frame))
+ while frames:
+ image_bytes = 0
+ for part, frame in frames:
+ encoded = io.BytesIO()
+ frame.save(encoded, format="JPEG", quality=quality, subsampling=0, optimize=True)
+ raw = encoded.getvalue()
+ image_bytes += len(raw)
+ part["image_url"]["url"] = "data:image/jpeg;base64," + base64.b64encode(raw).decode("ascii")
+ stats.update(image_quality=quality, sent_image_bytes=image_bytes)
+ if len(json.dumps(payload).encode("utf-8")) <= max_bytes or quality == 65:
+ break
+ quality = max(65, quality - 10)
+ while frames and len(json.dumps(payload).encode("utf-8")) > max_bytes and max(
+ max(frame.size) for _, frame in frames
+ ) > 960:
+ for part, frame in frames:
+ frame.thumbnail((max(960, int(frame.width * 0.8)), max(540, int(frame.height * 0.8))))
+ encoded = io.BytesIO()
+ frame.save(encoded, format="JPEG", quality=quality, subsampling=0, optimize=True)
+ part["image_url"]["url"] = "data:image/jpeg;base64," + base64.b64encode(
+ encoded.getvalue()).decode("ascii")
+ return {**stats, "unique_images": len(frames), "sent_image_sizes": [list(f.size) for _, f in frames]}
+
+
+
def extract_json_object(text: str):
"""从模型正文中抢救 JSON:剥围栏后,若剩余文本不是纯 JSON,抽取首个平衡的 {…}。
返回解析后的 dict;无 JSON 或解析失败返回 None。"""
@@ -75,6 +143,22 @@ def __init__(self, credential_file, config, audit_path: Path, llm_sink=None):
# llm_sink(role=..., request=..., response=..., **meta):全量请求/响应落盘钩子,
# 由 tandem_eval.recorder 提供(llm_calls.jsonl);None 时仅审计。
self.llm_sink = llm_sink
+ self.deadline = None
+
+ def set_deadline(self, deadline):
+ """Absolute monotonic task deadline, shared by requirements, planning and verification."""
+ self.deadline = deadline
+
+ def _remaining(self):
+ remaining = self.deadline - time.perf_counter() if self.deadline is not None else float("inf")
+ if remaining <= 0:
+ raise TaskBudgetExceeded("Task deadline exhausted during slow-model execution")
+ return remaining
+
+ def _retry_wait(self, seconds):
+ if seconds >= self._remaining():
+ raise TaskBudgetExceeded("Task budget cannot accommodate another slow-model retry")
+ time.sleep(seconds)
def json_request(self, messages, validate, *, role):
credentials = self.credentials
@@ -84,18 +168,35 @@ def json_request(self, messages, validate, *, role):
"model": credentials.get("model") or "unknown",
"messages": messages,
"stream": False,
- "max_tokens": self.config.slow_max_tokens,
+ "max_tokens": (min(self.config.slow_max_tokens, self.config.slow_verify_max_tokens)
+ if role == "verify_stage" else self.config.slow_max_tokens),
}
+ if role == "gui_choice":
+ payload["max_tokens"] = min(self.config.slow_max_tokens, 2048)
effort = self.config.slow_reasoning_effort
+ if role == "gui_choice":
+ effort = "low"
+ if role == "verify_stage" and self.config.slow_verify_reasoning_effort is not None:
+ effort = self.config.slow_verify_reasoning_effort
if effort == "off":
payload["enable_thinking"] = False
elif effort is not None:
payload["reasoning_effort"] = effort
+ image_stats = (_pack_images(payload, self.config.slow_image_quality,
+ self.config.slow_max_request_bytes)
+ if self.config.execution_mode == "goal" else {})
def build_request():
+ data = json.dumps(payload).encode("utf-8")
+ # Repairs can append messages after image packing; enforce the limit on every attempt.
+ if self.config.execution_mode == "goal" and len(data) > self.config.slow_max_request_bytes:
+ raise ValueError(
+ "slow model request body exceeds slow_max_request_bytes: "
+ f"{len(data)} > {self.config.slow_max_request_bytes} bytes"
+ )
return urllib.request.Request(
credentials["base_url"].rstrip("/") + "/chat/completions",
- data=json.dumps(payload).encode("utf-8"),
+ data=data,
headers={
**(
{"Authorization": "Bearer " + key["api_key"]}
@@ -125,6 +226,7 @@ def audit(event, started, attempt, **details):
"time": time.time(),
"elapsed_seconds": round(time.perf_counter() - started, 3),
"request_bytes": len(request.data),
+ **image_stats,
**details,
}
text = json.dumps(self.key_pool.redact(record), ensure_ascii=False)
@@ -152,13 +254,14 @@ def public_headers(headers):
attempt = 0
failures = 0
while True:
+ timeout = min(self.config.slow_timeout_seconds, self._remaining())
attempt += 1
started = time.perf_counter()
request = build_request() # 每次重试重建:升档思考强度需重新序列化
audit("request_start", started, attempt)
try:
try:
- with opener(request, timeout=self.config.slow_timeout_seconds) as response:
+ with opener(request, timeout=timeout) as response:
status = response.status
headers = public_headers(response.headers)
raw = json.load(response)
@@ -209,7 +312,7 @@ def public_headers(headers):
raise error
delay = retry_after_seconds(headers.get("retry-after"), 30)
audit("retry_wait", started, attempt, retry_wait_seconds=delay)
- time.sleep(delay)
+ self._retry_wait(delay)
continue
choices = raw.get("choices") or []
if not choices:
@@ -221,20 +324,23 @@ def public_headers(headers):
finish_reason=finish_reason, usage=raw.get("usage"))
if finish_reason == "length":
completion_tokens = (raw.get("usage") or {}).get("completion_tokens")
- raise ValueError(
+ raise ModelResponseError(
f"slow model response truncated by max_tokens (finish_reason=length, "
f"completion_tokens={completion_tokens}); raise slow_max_tokens"
)
if not content and choice.get("message", {}).get("reasoning_content"):
- raise ValueError(
+ raise ModelResponseError(
"slow model returned empty content (all completion tokens stayed in the "
"reasoning channel); escalating reasoning_effort for the next attempt"
)
value = extract_json_object(content)
if value is None:
- raise ValueError("slow model response contains no JSON object")
+ raise ModelResponseError("slow model response contains no JSON object")
if not validate(value):
- raise ValueError("slow model response is missing required fields")
+ correction = messages[-1].get("content", "") if messages else ""
+ detail = (correction if isinstance(correction, str)
+ and correction.startswith("Invalid") else "")
+ raise ModelResponseError("slow model response failed validation: " + detail[:600])
return value
except (urllib.error.URLError, ConnectionError, TimeoutError, ValueError) as error:
audit(
@@ -249,7 +355,15 @@ def public_headers(headers):
raise
# 内容级失败(空正文/截断/非法 JSON)时升档思考强度后重试,不原样重发
current_effort = payload.get("reasoning_effort")
- if current_effort in _REASON_EFFORT_LADDER:
+ if (self.config.execution_mode == "goal" and role == "plan_stage"
+ and "finish_reason=length" in str(error)):
+ payload["reasoning_effort"] = "low"
+ messages.append({"role": "user", "content":
+ "The previous response was truncated and NOTHING from it was executed. "
+ "Return one COMPLETE compact goal-stage JSON object, without reasoning or "
+ "a long script. Use one bounded edit/read, reuse memory, and preserve the "
+ "original requirements. Do not repeat the unbounded response."})
+ elif current_effort in _REASON_EFFORT_LADDER:
payload["reasoning_effort"] = _REASON_EFFORT_LADDER[current_effort]
audit("retry_wait", started, attempt, retry_wait_seconds=30)
- time.sleep(30)
+ self._retry_wait(30)
diff --git a/src/tandem/planning/goal_planner.py b/src/tandem/planning/goal_planner.py
new file mode 100644
index 0000000..473f37f
--- /dev/null
+++ b/src/tandem/planning/goal_planner.py
@@ -0,0 +1,370 @@
+"""One self-contained subgoal, not a speculative list of future UI actions."""
+
+import base64
+import json
+import re
+
+from ..memory.context import _fact_view, planning_context
+
+REQUIREMENTS = """Extract the ORIGINAL task into JSON {requirements:[{id,requirement,output_path,
+kind,units,unit_fact_keys,facts_needed}]}. kind is delivery or constraint. units is a list of STRINGS with
+delivery units with their completion criteria; enumerate explicitly requested pages/items, otherwise
+use one unit. Constraints have units:[]. facts_needed is [{key,description}] naming only essential
+stable source information acquired from supplied materials; reuse the SAME key across requirements.
+Never register current focus, open windows or selection as required source facts. Split large source
+packets into the concrete fields or small sections needed by individual deliverables. unit_fact_keys
+is an array parallel to units: each entry lists ONLY the facts_needed keys required by THAT unit.
+Keys may reference definitions in ANY requirement; define each shared source key once. Use [] for a
+unit needing no source reading. Constraints MUST have units:[] AND unit_fact_keys:[], not [[]].
+Include a constraint's required source key on the delivery units that must satisfy that constraint.
+A source-reading dependency must not gate unrelated work.
+Do not list facts already stated in the instruction as facts_needed. Names, titles, URLs and target
+systems explicitly provided by the user are inputs, not missing source facts.
+Keep all original quantities, locations, prohibitions, interaction requirements and saving conditions.
+Enumerate the individual pages/slides/form sections as small delivery units, including when their count
+is explicit but their names are not. Do not collapse 'all 26 slides' into one giant unit. Each unit
+lists only its own missing source content; whole-document consistency remains a final constraint.
+output_path is null unless literally given as an absolute output path in the instruction.
+Do not prescribe actions, guess source facts, inspect evaluators, or introduce hidden grading criteria.
+Return a compact contract, no reasoning, no speculative extra work."""
+
+REQUIREMENTS += '''
+Every requirement MUST contain ALL seven keys, including a plain-string "requirement" describing the
+actual obligation (also for constraints). units MUST be an array of STRINGS, never objects.
+Requirement IDs must be nonempty and UNIQUE across the complete contract, including constraints.
+Exact shape example: {"requirements":[{"id":"R1","requirement":"Create and save the requested document",
+"output_path":null,"kind":"delivery","units":["Requested document is complete and saved"],
+"unit_fact_keys":[[]],
+"facts_needed":[]}]}. This example illustrates the format only; extract the real task's obligations.
+'''
+
+PLAN = """Delegate ONE meaningful bounded subtask using the host's task memory. You are the planner;
+the fast GUI choice model will autonomously observe and act, not follow an action list. Return JSON:
+{done:false,work_item_id:"exact ready id",route:"gui|cli",purpose:"prepare|produce|save|verify",
+goal:"specific result",completion_criteria:"stage exit condition",method:"concise approach",
+input_fact_keys:["known key"],inputs:[{label:"field meaning",target_hint:"observed field label",
+text:"exact supplied text",
+input_mode:"replace|insert"}],guidance:"brief necessary navigation/context, not an ordered click script",
+max_actions:24,command:"CLI only; otherwise empty string",
+recovery_strategy:"continue|inspect_state|restore_context|alternate_interaction|alternate_route"}.
+When ready_work_items[].recovery_required is true, choose an explicit recovery_strategy other than
+continue. Changing the wording of a goal/method does not change the execution path. The host retains
+failed navigation families for the observed execution scope across stages; unrelated source facts do
+not release them. Restore the execution context, inspect a missing capability, use another interaction
+or switch route only when the original task permits it. Never repeat an already-executed action merely
+because a receipt or observation is missing.
+Use a ready work item's FIXED goal and original requirements. Do not rename units or lower final criteria.
+Missing source facts are acquisition targets, NOT a hard gate on starting a delivery unit. First make
+and save the useful portion supported by known facts; leave unknown values unresolved, never guess them.
+Choose a bounded incremental stage exit condition, not the whole unit's acceptance criteria. A partial
+stage may succeed while the unit remains incomplete. Read a missing source only before the next action
+actually needs it, then resume the preserved artifact. Respect explicit user-required reading order.
+input_fact_keys may name ONLY confirmed task-scope keys present in memory.known_facts, NOT keys that
+are merely requested in facts_needed. Use [] before any source facts have been observed.
+For prepare, acquire only missing essential facts; do not reread facts already in memory. Source notes
+are historical evidence, not guaranteed current UI. Reuse confirmed task facts. Temporary inventory,
+selection and page state must be reobserved. Do not fabricate a URL or missing personal information.
+Bound visual source reading to one small section at a time (normally 4-6 observations); the stage-end
+verifier will record its contents. Do not delegate reading several documents and navigating a new site
+as a single preparation goal. Later preparation continues from materials.covered_parts, not page one.
+For GUI give a whole local goal (a form section, adding an attachment, a settings operation), normally
+8-24 possible actions, not merely 'open menu'. Bound editing to the CURRENT page/form section or slide.
+Stop before a new page requires new values or reasoning; plan it from its actual observation next.
+Give only ONE current value per target_hint. Reading multiple URLs in the same address bar requires
+separate stages, not competing replacement inputs. A typed navigation URL still requires Enter.
+A file chooser may be completed with an already supplied exact path. Never delegate uploading while
+the current form still needs unknown required fields. First fill the known current fields, then stop
+to acquire only the missing values. Provide ALL exact text values from the instruction or known facts.
+For every text input, target_hint must identify its intended observed field (copy the visible label
+when available). For browser navigation use the observed address-bar label; for a document body or
+desktop launcher identify that specific editor/search field. Text is never interchangeable among fields.
+The choice model cannot generate free text. It can click, type provided inputs, use keyboard, scroll,
+wait, request stage verification, or return control. Missing data/new reasoning requires another stage.
+For CLI provide one bounded Bash command, current inputs and expected real results. Prefer actually
+producing/saving deliverables over repeatedly reading tutorials, building audits or protecting unchanged
+files. Inspect edited documents in the actual application before claiming them verified. Keep source
+and final paths distinct. CLI is forbidden if allow_cli=false. Task constraints override routing choices.
+Use installed tools; do not assume OCR binaries or passwordless sudo. Do not install packages as a
+substitute for source reading. Keep commands valid Bash, including background-command separators.
+memory.observed_missing_modules records actual import failures: use an available alternative, not the
+same missing package again. Derive edit preconditions from observed document structure, not guessed
+replacement counts. After an assertion fails, narrow the next edit using the real error and partial output.
+Do not inspect evaluator code, ground truth, rewards, hidden state, unrelated secrets or scoring APIs.
+Previous methods/errors are in memory; changing their description does not make them new. If stalled,
+choose another method or another independent ready unit. Avoid optional research and cosmetic detours.
+Only return {done:true,task_status:"completed|infeasible",current_goal:"evidence-based reason"} for an
+actual whole-task terminal claim. Stage completion is NOT task completion. A blocked unit is not proof
+the task is infeasible. Keep the response short: goal, inputs and one bounded command, no long analysis."""
+
+PLAN += """
+Treat capability_missing and unresolved execution scopes as prerequisites, not wording problems.
+memory.pending_publications are edited working copies not yet delivered. Publish a useful saved increment
+to the observed user deliverable and reopen it before starting more optional research or another edit.
+Saving /tmp files is not delivery. After a bounded edit, prefer publishing that increment rather than
+spending the remaining budget on tiny edits in an unpublished copy. Do not overwrite it from a stale GUI.
+Keep a valuable unsaved form in its own tab when opening a reference; explicitly preserve that context.
+Use remaining_wall_seconds as a real deadline. Publish a useful current increment early; near the deadline,
+prefer final-path saving/reopening over optional source exploration or another tiny unpublished edit.
+Partial work is not task completion: retain unmet requirements and let official evaluation judge it.
+Opening a file means it is loaded and usable in the requested application. Complete necessary native
+open/import confirmations; handing off a path or leaving an import preview is not the finished handoff.
+memory.active_blocker is the current observed execution obstruction, shared across work units.
+Changing its name, unit or recovery_strategy does not resolve it. GUI recovery must expose an actually
+available target/interaction; after grounding is exhausted use existing observed controls or another
+allowed route. No new speculative coordinates or repeated requests for the exhausted capability.
+Give goal as an action-oriented bounded objective, and completion_criteria as a SEPARATE desired
+observable outcome. An exit condition is not an observed fact. Do not invent folders or navigation
+capabilities: a page number printed in source content does not establish a viewer pagination control.
+Resolve the active obstruction in ONE bounded prepare stage before proposing edits behind it. A business
+dialog needed for the task must be completed, not dismissed. Do not reissue an unbound input unchanged:
+first navigate to expose its target, acquire a supported interaction, or use another allowed route.
+After CLI edits, the already-open GUI may be stale. Close/reopen the saved working artifact without
+overwriting it from the old GUI buffer before visual acceptance. A title alone does not establish freshness.
+Prefer a ready delivery unit over more preparation. Preparation is bounded acquisition for ONE missing
+field or source section, not an all-document audit or a pixel-analysis research project. State the exact
+observable subgoal, never 'Acquire the required source facts'. Reuse saved extractions and artifacts.
+After two stages without new required facts or delivery progress, use a genuinely different method or
+another ready unit. Do not add optional diagnostic outputs as new completion requirements. Make and
+save a usable increment as soon as its inputs are known; preserve all original final requirements.
+historical_ui_notes are prior observations, NOT the current interface. Current screenshot/controls win.
+For prepare, follow next_acquisition: acquire the stated missing registered fields for that ONE unit.
+Do not inventory every dropdown or inspect source code before filling fields whose values are known.
+Use exact registered keys for acquired contents; fact_index lists already observed auxiliary sources,
+not new dependencies. Do not reread auxiliary sources solely because their full text is omitted here.
+Never type into an unrelated/Untitled document. Confirm the target application and document first.
+For editing an existing document, include target_document:"exact filename" on GUI produce/save stages.
+If it is not currently open in the foreground, first use a prepare stage to open/switch to it. A
+navigation-only prepare stage may have no source fact targets. For websites or creating a NEW document,
+target_document may be null. Available GUI operations include right-click, double-click, Escape,
+Alt+Tab, Ctrl+O, keyboard navigation, supplied text, scrolling and screenshot-grounded targets.
+known_facts includes partial and auxiliary source content; complete:false is usable for a partial
+increment but does NOT mean the source or unit is complete. Never guess ownership/identity when joining
+records from different sources. Do not keep preparing solely to make every facts_needed entry complete.
+If an observed fact is indexed but its text is omitted/excerpted, return ONLY {read_memory:["key",...]}
+for up to six keys, at most twice before planning. This reads stored evidence without reopening files.
+Failure constraints, remaining work and rejected memory updates are actionable state, not suggestions
+to repeat the same attempt with different wording. When the executor reports a blocked cycle, choose
+another interaction, another source or an independent ready unit.
+For a GUI stage that needs to open or switch applications, include target_application:"exact app name".
+This explicit input will also be available for typing in the desktop launcher; do not leave it only in
+prose guidance. Set it to null when not applicable. Prefer CLI to open an installed application or a
+known document when allowed, then delegate actual UI editing to JEV. Do not alternate the application
+drawer open/closed or repeat a covered target. Resolve the visible overlay or use another interaction.
+completed milestones are delivered sections, not instructions to recreate them. Continue with an unmet
+section/field group even when the whole unit is incomplete; do not re-audit completed increments.
+For native input_type=date, provide ISO YYYY-MM-DD from the task/source. The executor edits the observed
+Day/Month/Year segments separately; never concatenate a guessed locale's month/day/year digits.
+When a date is still empty after typing, use its calendar controls or revise the input; don't retype it.
+Two consecutive CLI execution failures require one bounded read-only structure inspection or GUI work
+before another write. Use actual object identities; do not weaken assertions to obtain a pass.
+memory.artifacts is the cumulative working-copy registry: resume its existing working_path for the
+same target_path, not a fresh copy of the original on each stage. Save increments to that same artifact.
+Do not replace editable slides, animations or native objects with full-page reference screenshots.
+Treat website application code as read-only unless the user explicitly asks to develop that application.
+Do not edit HTML/JS source, inject bootstraps, add options, remove fields or bypass validation to fill
+a website. Use its real UI; ordinary CLI editing of the requested output document remains allowed.
+For a supplied command entered into an application's command/input bar, add kind:"command",commit:"enter"
+to that input. Ordinary text uses kind:"text",commit:null; a known selection uses kind:"choice".
+These describe one observed control interaction, not a sequence of speculative future clicks.
+unsupported_methods records ACTUAL application errors, not suggestions. Do not keep changing arguments
+of an unrecognized command; use another supported interaction in the same required application.
+Memory failures.error is observed evidence; suggested_actions is only a hypothesis. Check it against
+the real application before treating it as valid syntax. Preserve already confirmed source content.
+Window opening/maximization/menu navigation are prepare, not produce or saved-delivery milestones.
+Keep the stage exit condition separate from the fixed unit criterion and whole-task requirements.
+"""
+
+
+def goal_requirements(client, instruction):
+ messages = [{"role": "system", "content": REQUIREMENTS}, {"role": "user", "content": instruction}]
+
+ def valid(v):
+ okay = (isinstance(v.get("requirements"), list) and bool(v["requirements"])
+ and all(isinstance(r, dict) and isinstance(r.get("id"), str)
+ and isinstance(r.get("requirement"), str)
+ and r.get("kind") in {"delivery", "constraint"}
+ and isinstance(r.get("units"), list)
+ and (r.get("kind") == "constraint" or bool(r["units"]))
+ and all(isinstance(u, str) for u in r["units"])
+ and isinstance(r.get("unit_fact_keys"), list)
+ and len(r["unit_fact_keys"]) == len(r["units"])
+ and all(isinstance(keys, list) and all(isinstance(k, str) for k in keys)
+ for keys in r["unit_fact_keys"])
+ and isinstance(r.get("facts_needed"), list)
+ and all(isinstance(f, dict) and isinstance(f.get("key"), str)
+ and isinstance(f.get("description"), str) for f in r["facts_needed"])
+ for r in v["requirements"]))
+ if okay:
+ ids = [r["id"].strip() for r in v["requirements"]]
+ if not all(ids) or len(set(ids)) != len(ids):
+ messages.append({"role": "user", "content":
+ "Duplicate or empty requirement id. Give EVERY requirement a unique nonempty id; "
+ "preserve ALL delivery obligations and return the COMPLETE corrected contract."})
+ return False
+ registered = {f["key"] for r in v["requirements"] for f in r["facts_needed"]}
+ okay = all(set(keys) <= registered for r in v["requirements"] for keys in r["unit_fact_keys"])
+ if not okay:
+ messages.append({"role": "user", "content":
+ "Invalid requirements JSON. Every entry needs id, requirement (plain text, mandatory), "
+ "output_path, kind, units (array of STRINGS, not objects), unit_fact_keys "
+ "(one array of source key strings per unit) and facts_needed "
+ "(array of {key,description}). Include constraint text in requirement; units:[] is not "
+ "a substitute. Constraints need units:[] AND unit_fact_keys:[], not [[]]. "
+ "Shared source keys may be defined in any requirement. Return the COMPLETE object "
+ "{requirements:[...]}, not an isolated fact or requirement."})
+ return okay
+
+ value = client.json_request(messages, valid, role="task_requirements")
+ for r in value["requirements"]:
+ path = r.get("output_path")
+ if not isinstance(path, str) or not path.startswith("/") or path not in instruction:
+ r["output_path"] = None
+ return value["requirements"]
+
+
+def goal_error(value, memory, allow_cli):
+ """Name the rejected field/constraint so JSON repair can actually change the request."""
+ if type(value.get("done")) is not bool:
+ return "done must be a boolean"
+ if value["done"]:
+ return ("" if value.get("task_status") in ("completed", "infeasible")
+ and isinstance(value.get("current_goal"), str) and value["current_goal"].strip()
+ else "terminal claims require task_status and a nonempty current_goal")
+ ready = {i["id"]: i for i in memory.ready_items()}
+ if not isinstance(value.get("work_item_id"), str):
+ return "work_item_id must be a string naming an exact ready unit"
+ item = ready.get(value.get("work_item_id"))
+ if item is None:
+ return "work_item_id must be one of " + json.dumps(list(ready))
+ strategy = value.get("recovery_strategy", "continue")
+ if strategy not in {"continue", "inspect_state", "restore_context", "alternate_interaction",
+ "alternate_route"}:
+ return "recovery_strategy must name a supported execution recovery"
+ if item.get("recovery_required") and strategy == "continue":
+ return "execution recovery is required; declare how the blocked path will change"
+ if item.get("stalled_stages", 0) >= 2:
+ previous = item.get("last_attempt", {})
+ if (value.get("route") == previous.get("route") == "cli"
+ and isinstance(value.get("command"), str)
+ and value["command"].strip() == previous.get("command", "").strip()):
+ return "command repeats the same CLI attempt after two stalled stages; change method or work unit"
+ if value.get("route") not in (("gui", "cli") if allow_cli else ("gui",)):
+ return "route must be gui" + (" or cli" if allow_cli else "; CLI is disabled")
+ purposes = ("prepare",) if item["kind"] == "prepare" else ("prepare", "produce", "save", "verify")
+ if value.get("purpose") not in purposes:
+ return "purpose must be one of " + json.dumps(sorted(purposes))
+ if (item.get("failed_cli_stages", 0) >= 2 and value.get("route") == "cli"
+ and value.get("purpose") in {"produce", "save"}):
+ return ("two CLI executions failed; inspect structure read-only in prepare "
+ "or use GUI before writing")
+ for key in ("goal", "completion_criteria", "method"):
+ if not isinstance(value.get(key), str) or not value[key].strip():
+ return key + " must be a nonempty string"
+ if not isinstance(value.get("guidance"), str):
+ return "guidance must be a string (empty allowed)"
+ for key in ("target_document", "target_application"):
+ if value.get(key) is not None and (not isinstance(value[key], str) or not value[key].strip()):
+ return key + " must be a nonempty string or null"
+ app_words = re.findall(r"\w{3,}", (value.get("target_application") or "").casefold())
+ blockers = [c for c in memory.execution_constraints if c.get("kind") == "execution_checkpoint"
+ and c.get("global_scope") and c.get("recovery_required")
+ and (not app_words or any(w in json.loads(c["scope_key"])[0].casefold() for w in app_words))]
+ if blockers and value.get("route") == "gui" and value.get("purpose") in {"produce", "save"}:
+ return "resolve the active GUI input obstruction in a bounded prepare stage before editing"
+ if type(value.get("max_actions")) is not int or not 1 <= value["max_actions"] <= 24:
+ return "max_actions must be an integer in 1..24"
+ allowed = [k for k, f in memory.facts.items() if f["status"] == "confirmed" and f["scope"] == "task"]
+ if not isinstance(value.get("input_fact_keys"), list) or any(
+ not isinstance(k, str) or k not in allowed for k in value["input_fact_keys"]):
+ return ("input_fact_keys must contain only acquired confirmed task keys: " + json.dumps(allowed)
+ + "; copy keys EXACTLY, including dots and punctuation; do not normalize filenames. "
+ "Use [] if none are needed. Rejected keys: " + json.dumps(value.get("input_fact_keys")))
+ if not isinstance(value.get("inputs"), list):
+ return "inputs must be an array (empty allowed)"
+ input_values = {}
+ for index, entry in enumerate(value["inputs"]):
+ if (not isinstance(entry, dict) or not isinstance(entry.get("label"), str)
+ or not isinstance(entry.get("text"), str)
+ or entry.get("input_mode") not in ("replace", "insert")):
+ return f"inputs[{index}] requires label:string, text:string, input_mode:replace|insert"
+ if "target_hint" in entry and (not isinstance(entry["target_hint"], str)
+ or not entry["target_hint"].strip()):
+ return f"inputs[{index}].target_hint must identify the intended field"
+ if entry.get("kind", "text") not in {"text", "command", "choice"}:
+ return f"inputs[{index}].kind must be text, command or choice"
+ if entry.get("commit") not in (None, "enter"):
+ return f"inputs[{index}].commit must be enter or null"
+ blocked = memory.blocked_command(entry["text"], value.get("target_application"))
+ if blocked and value["route"] == "gui":
+ return blocked
+ hint = entry.get("target_hint", entry["label"]).strip().casefold()
+ if value["route"] == "gui" and hint in input_values and input_values[hint] != entry["text"]:
+ return f"inputs assign conflicting texts to {hint!r}; delegate one current value/URL per stage"
+ input_values[hint] = entry["text"]
+ previous = item.get("last_attempt", {})
+ if (item.get("stalled_stages", 0) >= 2
+ and previous.get("failure_kind", "").startswith("capability_missing:")
+ and value["route"] == "gui" and value["inputs"]
+ and [i["text"] for i in value["inputs"] if i.get("scope") != "desktop_launcher"]
+ == [i["text"] for i in previous.get("inputs", []) if i.get("scope") != "desktop_launcher"]):
+ return "the same input is unsupported; expose its target in a prepare stage or change route"
+ if not isinstance(value.get("command"), str):
+ return "command must be a string; use an empty string for GUI"
+ if value["route"] == "cli" and not value["command"].strip():
+ return "command must be a nonempty Bash command for CLI"
+ return ""
+
+
+def valid_goal(value, memory, allow_cli):
+ return not goal_error(value, memory, allow_cli)
+
+
+def plan_goal(client, instruction, screenshot, targets, recent, tree, verified_focus=None,
+ requirements=None, remaining_steps=None, *, memory, allow_cli=True, route_policy="auto",
+ remaining_seconds=None):
+ allow_cli = allow_cli and not re.search(
+ r"do not (?:open or use |use |open )?(?:terminal|the terminal)|\bGUI[- ]only\b|\bCLI is forbidden\b",
+ instruction, re.I)
+ context = {"task": instruction, "memory": planning_context(memory), "allow_cli": bool(allow_cli),
+ "route_policy": route_policy, "remaining_actions": remaining_steps,
+ "remaining_wall_seconds": remaining_seconds,
+ "visible_controls": [t.compact() for t in targets], "recent": recent[-4:]}
+ messages = [{"role": "system", "content": PLAN}, {"role": "user", "content": [
+ {"type": "text", "text": json.dumps(context, ensure_ascii=False)},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,"
+ + base64.b64encode(screenshot.read_bytes()).decode("ascii")}},
+ ]}]
+ reads = 0
+
+ def validate(v):
+ if set(v) == {"read_memory"}:
+ okay = (reads < 2 and isinstance(v["read_memory"], list) and 1 <= len(v["read_memory"]) <= 6
+ and all(isinstance(k, str) and k in memory.facts and memory.facts[k]["scope"] == "task"
+ for k in v["read_memory"]))
+ error = ("read_memory needs 1..6 existing task-scope keys and at most two reads"
+ if not okay else "")
+ else:
+ error = goal_error(v, memory, allow_cli)
+ if error:
+ messages.append({"role": "user", "content":
+ "Invalid goal-stage JSON: " + error + ". Return the COMPLETE corrected compact stage "
+ "object, preserving evidence and original requirements. Do not replay an executed action."})
+ return not error
+
+ while True:
+ value = client.json_request(messages, validate, role="plan_stage")
+ if set(value) != {"read_memory"}:
+ application = value.get("target_application")
+ if not value["done"] and value["route"] == "gui" and application and not any(
+ i["text"] == application for i in value["inputs"]):
+ value["inputs"].append({"label": "Target application name for desktop search",
+ "target_hint": "Desktop search", "text": application,
+ "input_mode": "replace", "scope": "desktop_launcher"})
+ return value
+ reads += 1
+ messages.append({"role": "assistant", "content": json.dumps(value)})
+ messages.append({"role": "user", "content": json.dumps({
+ "stored_source_facts": [_fact_view(memory.facts[k], 16000) for k in value["read_memory"]],
+ "instruction": "Use the stored evidence; now return a bounded stage plan. "
+ "Do not reopen these sources just to recover the same content.",
+ "memory_reads_remaining": 2 - reads}, ensure_ascii=False)})
diff --git a/src/tandem/runtime/episode.py b/src/tandem/runtime/episode.py
index bb4dbeb..73efc3c 100644
--- a/src/tandem/runtime/episode.py
+++ b/src/tandem/runtime/episode.py
@@ -12,6 +12,7 @@
from pathlib import Path
from ..gui.focus import action_key, toggle_transition
+from ..memory.context import verification_context
from ..protocol.models import CliAction, FailAction, FinishAction, Observation
from .stopping import NO_ACTION_LIMIT
@@ -75,6 +76,7 @@ def terminal_decision(session, plan, observation: Observation, tree, output_file
control_state_check=unchanged,
requirements=session.requirements,
output_files=output_files,
+ **({"task_memory": verification_context(session.goal.memory)} if session.goal is not None else {}),
)
session.events.append(
{
diff --git a/src/tandem/runtime/goal_stage.py b/src/tandem/runtime/goal_stage.py
new file mode 100644
index 0000000..6af3ad5
--- /dev/null
+++ b/src/tandem/runtime/goal_stage.py
@@ -0,0 +1,494 @@
+"""Goal-stage loop: receipts/observations each step, one semantic verifier at stage end."""
+
+import json
+import re
+import time
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+from ..gui.goal_candidates import (
+ application_mismatch,
+ foreground_tree,
+ goal_candidates,
+ launcher_input,
+ portal_needs_grounding,
+)
+from ..gui.goal_candidates import (
+ document_mismatch as _document_mismatch,
+)
+from ..gui.interaction import target_effect as _target_effect
+from ..gui.interaction import visual_changed as _visual_changed
+from ..gui.interaction import visual_identity
+from ..memory.context import execution_context
+from ..models.ranking import ranking_dict
+from ..models.slow import ModelResponseError
+from ..protocol.models import CliAction, FailAction
+from .episode import handle_done, resolve_done_probe
+
+
+class ObservationRequired(Exception):
+ """Refresh without executing/replaying an action or consuming an action budget."""
+
+
+def _state_key(targets):
+ return tuple(sorted((t.role, t.name, t.x, t.y, t.value, bool(t.focused), str(t.checked), str(t.selected))
+ for t in _progress_targets(targets)))
+
+
+def _progress_targets(targets):
+ # Shell display text (clock/battery) is not task progress. Keep shell controls and focused input.
+ return [t for t in targets if not (
+ (t.application == "gnome-shell" or "gnome-shell" in t.parent_context)
+ and t.role in {"text", "label", "static"} and not t.focused)]
+
+
+def _loop_state(targets, tree):
+ root = ET.fromstring(tree or "")
+ return (root.get("foreground_title", ""), root.get("foreground_class", ""),
+ tuple(sorted((t.role, t.semantic_key if "dom" in t.source else _control_name(t),
+ 0 if "dom" in t.source else t.x, 0 if "dom" in t.source else t.y,
+ "" if t.role == "slider" else t.value,
+ str(t.checked)) for t in _progress_targets(targets)
+ if t.application != "gnome-shell" and "gnome-shell" not in t.parent_context)),
+ portal_needs_grounding(tree))
+
+
+def _control_name(target):
+ if target.role != "page_tab":
+ return target.name
+ name = re.sub(r"\s+- Memory usage - .+$", "", target.name)
+ return re.sub(r"\s*[–-]\s*(?:Audio playing|Video playing in picture-in-picture mode)$", "", name)
+
+
+def _cycle_actions(recent, state):
+ for period in (2, 3, 4):
+ if len(recent) < period * 2:
+ continue
+ first, second = recent[-period * 2:-period], recent[-period:]
+ if first == second and any(a != first[0] for a in first):
+ return {a["key"] for a in second if a["state"] == state}
+ return set()
+
+
+def _action_key(candidate, targets=()):
+ target = next((t for t in targets if t.id == candidate.target_id), None)
+ name = ((target.semantic_key if "dom" in target.source else
+ f"{target.role}|{_control_name(target)}|{target.x}|{target.y}") if target else
+ visual_identity(candidate) or candidate.metadata.get("semantic_key") or candidate.description)
+ if candidate.operation == "keypress":
+ name += repr([(t.role, t.name, t.x, t.y) for t in targets if t.focused])
+ parameters = {k: candidate.metadata[k]
+ for k in ("text", "input_mode", "fraction", "key_sequence", "visual_point")
+ if k in candidate.metadata and not (k == "visual_point" and visual_identity(candidate))}
+ return (candidate.operation, name, json.dumps(parameters, sort_keys=True, ensure_ascii=False))
+
+
+def _navigation_key(candidate, target, work, state):
+ if candidate.operation == "scroll" or candidate.id in {"KEYPRESS_PAGE_UP", "KEYPRESS_PAGE_DOWN"}:
+ return (work, state[1], "browse", "up" if "UP" in candidate.id.upper() else "down")
+ if candidate.source == "a11y_menu":
+ return (work, state[1], "menu_navigation", candidate.description)
+ if target and target.scope == "browser_chrome" and candidate.operation in {"click", "double_click"}:
+ return (work, state[1], "browser_navigation", target.name.casefold())
+ if target and (target.application == "gnome-shell" or "gnome-shell" in target.parent_context):
+ return (work, state[1], "shell_navigation", "system_controls")
+ if target and re.search(r"\b(?:zoom|pan)\b", target.name, re.I):
+ return (work, state[1], candidate.operation, target.name.casefold())
+ return None
+
+
+class GoalExecution:
+ def __init__(self, memory, max_actions, interaction):
+ self.memory = memory
+ self.max_actions = max_actions
+ self.interaction = interaction
+ self.pending = None
+ self.history = []
+ self.end_reason = None
+ self.visual = []
+ self.visual_frame = None
+ self.recovery_stage = None
+ self.recovery_calls = 0
+ self.recovery_seconds = 0.0
+ self.attempts = []
+ self.failed_actions = {}
+ self.grounded_states = {}
+ self.no_effect = 0
+ self.empty_stages = 0
+
+ def _emit_state(self, session, step):
+ session.events.append({"route": "task_memory", "step": step,
+ "state": self.memory.snapshot()})
+
+ def _receive(self, session, observation, targets, tree):
+ if self.pending is None:
+ return
+ receipt = observation.last_receipt
+ if receipt is None or not receipt.acknowledged:
+ raise RuntimeError("Missing execution receipt; action is not replayed")
+ session.consecutive_no_action = 0
+ old = self.pending
+ self.pending = None
+ signature = _state_key(targets)
+ changed = old["kind"] == "cli" or signature != old["state"] or _visual_changed(
+ old["before_screenshot"], observation.screenshot.path)
+ if old.get("effect_bounds") and old["operation"] == "type_text":
+ changed = changed or _visual_changed(old["before_screenshot"], observation.screenshot.path,
+ old["effect_bounds"])
+ effect = _target_effect(old, targets)
+ if effect == "met":
+ self.recovery_calls, self.recovery_seconds = 0, 0.0
+ relevant_change = (old["kind"] == "cli" or effect == "met"
+ or effect != "not_met" and old["loop_state"] != _loop_state(targets, tree))
+ if effect == "unknown" and (not targets or old.get("visual_action")):
+ relevant_change = changed
+ if old.get("overlay_bounds") and effect != "met":
+ relevant_change = _visual_changed(old["before_screenshot"], observation.screenshot.path,
+ old["overlay_bounds"])
+ self.interaction.receive(old, effect, changed, receipt.returncode, targets=targets,
+ evidence_ref=f"A{observation.step_idx}")
+ self.memory.active_stage["interaction"] = self.interaction.snapshot()
+ if old["kind"] != "cli":
+ key = (self.memory.active_stage["work_item_id"], old["loop_state"], old["candidate_key"])
+ if not relevant_change:
+ self.failed_actions[key] = self.failed_actions.get(key, 0) + 1
+ self.attempts.append({"work": key[0], "state": old["loop_state"], "key": old["candidate_key"]})
+ self.attempts = self.attempts[-16:]
+ self.no_effect = 0 if relevant_change else self.no_effect + 1
+ record = {"step": observation.step_idx, "evidence_ref": f"A{observation.step_idx}",
+ "kind": old["kind"], "operation": old["operation"], "target": old["target"],
+ "action": old["action"], "returncode": receipt.returncode,
+ "stdout": receipt.stdout, "stderr": receipt.stderr,
+ "full_output_ref": receipt.full_output_ref, "screenshot": observation.screenshot.path,
+ "observed_controls": [t.compact() for t in targets][:40],
+ "source_frame": self.memory.active_stage["purpose"] == "prepare",
+ "signature": signature, "observed_change": changed,
+ "relevant_change": relevant_change, "target_effect": effect,
+ "interaction_changed": changed, "local_effect": self.interaction.local_effect,
+ "interaction": self.interaction.snapshot(),
+ "application": ET.fromstring(tree or "").get("foreground_class", ""),
+ "navigation_key": old.get("navigation_key")}
+ self.history.append(record)
+ self.memory.record_action(record)
+ session.recent.append({"operation": old["operation"], "target": old["target"],
+ "goal": self.memory.active_stage["goal"],
+ "outcome": "changed" if relevant_change else "no_effect",
+ "returncode": receipt.returncode,
+ "evidence_ref": record["evidence_ref"]})
+ session.events.append({"route": "goal_action", "step": observation.step_idx,
+ "record": record, "effect_verified": False})
+ if receipt.returncode != 0:
+ self.end_reason = "execution_error"
+ elif old["kind"] == "cli":
+ self.end_reason = "cli_returned"
+ elif self.memory.active_stage["actions"] >= self.memory.active_stage["max_actions"]:
+ self.end_reason = "stage_action_limit"
+ elif self.interaction.checkpoint_due:
+ self.end_reason = "execution_checkpoint_stalled"
+ elif self.no_effect >= 3 and (session.components.grounder is None
+ or (self.memory.active_stage["work_item_id"], _loop_state(targets, tree))
+ in self.grounded_states):
+ self.end_reason = "repeated_action_without_observable_progress"
+
+ def _finish(self, session, observation):
+ reference = f"{self.memory.active_stage['id']}:O{observation.step_idx}"
+ evidence = {h["evidence_ref"]: h for h in self.history}
+ evidence[reference] = {"screenshot": observation.screenshot.path}
+ started = time.perf_counter()
+ result = session.components.stage_verifier(
+ session.instruction, Path(observation.screenshot.path), memory=self.memory,
+ history=[{k: v for k, v in h.items() if k != "signature"} for h in self.history],
+ end_reason=self.end_reason, evidence=evidence)
+ stage_id = self.memory.active_stage["id"]
+ work = self.memory.active_stage["work_item_id"]
+ source_progress = (self.memory.stats["new_facts"], self.memory.stats["source_parts"])
+ delivery = (self.memory.stats["delivery_advances"], self.memory.stats["milestone_advances"])
+ self.interaction.finish(self.memory, result, self.history)
+ advanced = self.memory.apply(result, evidence)
+ acquired = source_progress != (self.memory.stats["new_facts"], self.memory.stats["source_parts"])
+ delivered = delivery != (self.memory.stats["delivery_advances"],
+ self.memory.stats["milestone_advances"])
+ if delivered:
+ self.recovery_calls, self.recovery_seconds = 0, 0.0
+ self.empty_stages = self.empty_stages + 1 if not self.history and not advanced and not acquired else 0
+ if self.memory.work_items[work]["status"] == "done":
+ self.failed_actions = {k: v for k, v in self.failed_actions.items() if k[0] != work}
+ self.attempts = [a for a in self.attempts if a["work"] != work]
+ self.memory.execution_constraints = [c for c in self.memory.execution_constraints
+ if c["work_item_id"] != work or c.get("global_scope")]
+ for rejected in self.memory.last_rejections:
+ session.events.append({"route": "memory_update_rejected", "stage_id": stage_id, **rejected})
+ session.events.append({"route": "stage_verified", "step": observation.step_idx,
+ "stage_id": stage_id, "end_reason": self.end_reason,
+ "evidence": {k: {"screenshot": v["screenshot"],
+ "full_output_ref": v.get("full_output_ref")}
+ for k, v in evidence.items()},
+ "verification": result, "memory_advanced": advanced,
+ "progress_delta": {"source": acquired, "delivery": delivered},
+ "control_decision": "completed_unit" if self.memory.work_items[work]["status"]
+ == "done" else "recover_execution" if self.memory.work_items[work].get(
+ "recovery_required") else "continue_increment" if advanced else "replan",
+ "seconds": time.perf_counter() - started})
+ session.recent.append({"operation": "stage_verified", "stage_id": stage_id,
+ "outcome": result["stage_outcome"], "evidence": result["summary"],
+ "remaining": result["remaining"]})
+ self.history, self.end_reason, self.visual = [], None, []
+ self.visual_frame = None
+ self.no_effect = 0
+ self._emit_state(session, observation.step_idx)
+ if result.get("failure_kind") == "policy_violation":
+ session.events.append({"route": "goal_policy_violation", "step": observation.step_idx,
+ "reason": result["policy_violation"]})
+ return FailAction(reason="policy_violation: " + result["policy_violation"])
+ if self.empty_stages >= 3:
+ session.events.append({"route": "goal_stalled", "step": observation.step_idx,
+ "reason": "three_consecutive_stages_without_an_action",
+ "blocking_reason": result.get("failure_kind", "")})
+ return FailAction(reason="three_consecutive_stages_without_an_action")
+ raise ObservationRequired("stage_verification_returned")
+
+ def act(self, session, observation):
+ tree = session._read_tree(observation)
+ if tree:
+ tree = foreground_tree(tree)
+ screenshot = Path(observation.screenshot.path)
+ targets = session.components.target_builder.build(
+ tree, f"t{observation.step_idx}", 128, infer_button_labels=True, include_sliders=True,
+ context=" ".join(str(v) for k, v in (self.memory.active_stage or {}).items()
+ if k in {"goal", "completion_criteria", "method", "guidance"}
+ ) or session.instruction, filter_occluded=True,
+ ) if tree else []
+ targets = session.components.semantic_enricher(
+ targets, tree, (observation.screenshot.width, observation.screenshot.height),
+ session._read_browser_snapshot(observation))
+ self._receive(session, observation, targets, tree)
+ if self.end_reason:
+ return self._finish(session, observation)
+ if session.done_probe is not None:
+ result = resolve_done_probe(session, observation, tree)
+ if result is not None:
+ return result
+ raise ObservationRequired("terminal_rejected")
+ if self.memory.active_stage is None:
+ started = time.perf_counter()
+ try:
+ plan = session.components.planner(
+ session.instruction, screenshot, targets, session.recent, tree or "",
+ requirements=session.requirements, remaining_steps=observation.budget.steps_remaining,
+ memory=self.memory, remaining_seconds=observation.budget.wall_seconds_remaining)
+ except ModelResponseError as error:
+ session.events.append({"route": "planner_output_invalid", "step": observation.step_idx,
+ "reason": str(error)})
+ return FailAction(reason="planner_output_invalid")
+ session.events.append({"route": "planner", "step": observation.step_idx,
+ "mode": "goal", "plan": plan,
+ "seconds": time.perf_counter() - started})
+ if plan["done"]:
+ result = handle_done(session, plan, observation, tree)
+ if result is not None:
+ return result
+ raise ObservationRequired("terminal_rejected")
+ plan["max_actions"] = min(plan["max_actions"], self.max_actions)
+ self.memory.start(plan, observation.step_idx)
+ self._emit_state(session, observation.step_idx)
+ raise ObservationRequired("planner_returned")
+ stage = self.memory.active_stage
+ document = stage.get("target_document")
+ stage["document_recovery"] = bool(document and _document_mismatch(document, tree, targets))
+ app = stage.get("target_application")
+ stage["application_recovery"] = bool(app and application_mismatch(app, tree))
+ if stage["route"] == "cli":
+ action = CliAction(command=stage["command"], timeout_s=session.config.cli_timeout_seconds,
+ want_observation="screenshot")
+ self.pending = {"kind": "cli", "operation": "cli", "target": stage["method"],
+ "action": action.to_dict(), "state": _state_key(targets),
+ "before_screenshot": observation.screenshot.path}
+ session.events.append({"route": "cli", "step": observation.step_idx,
+ "stage_id": stage["id"], "action": action.to_dict()})
+ return action
+ return self._gui(session, observation, targets, tree, screenshot)
+
+ def _gui(self, session, observation, targets, tree, screenshot):
+ context = execution_context(self.memory)
+ stage = self.memory.active_stage
+ state = _loop_state(targets, tree)
+ work = stage["work_item_id"]
+ cached = self.grounded_states.get((work, state))
+ if cached and (cached[1] != stage["id"] or _visual_changed(cached[0], screenshot)):
+ self.grounded_states.pop((work, state))
+ session.events.append({"route": "recovery_cache_invalidated", "step": observation.step_idx,
+ "stage_id": stage["id"], "reason": "scene_or_subgoal_changed"})
+ if self.visual and not self.interaction.reusable_visual(self.visual_frame, state, screenshot):
+ self.visual, self.visual_frame = [], None
+ self.grounded_states.pop((work, state), None)
+ session.events.append({"route": "recovery_invalidated", "step": observation.step_idx,
+ "stage_id": stage["id"], "reason": "last_action_invalidated_coordinates"
+ if self.interaction.invalidate_visual else "task_scene_changed"})
+ recent = [a for a in self.attempts if a["work"] == work][-8:]
+ cycle_keys = _cycle_actions(recent, state)
+ cycle = bool(cycle_keys)
+ blocked = {key[2] for key, count in self.failed_actions.items()
+ if key[:2] == (work, state) and count >= 2}
+ if cycle:
+ blocked.update(cycle_keys)
+ for key in blocked:
+ failure_key = (work, state, key)
+ self.failed_actions[failure_key] = max(2, self.failed_actions.get(failure_key, 0))
+ for key in blocked:
+ constraint = {"work_item_id": work, "window": list(state[:2]), "action_key": list(key),
+ "reason": "Repeated action/cycle in the same observed controls. Try a different "
+ "interaction; unrelated animation or new facts do not clear it."}
+ if constraint not in self.memory.execution_constraints:
+ self.memory.execution_constraints.append(constraint)
+ context["avoid_actions"] = [list(key) for key in blocked]
+ semantic_missing = (not any("gnome-shell" not in t.parent_context for t in targets)
+ or bool(portal_needs_grounding(tree)))
+ space = session.components.candidate_builder.build(
+ targets, 96, action_inputs=[], context=" ".join(
+ context[k] for k in ("goal", "completion_criteria", "method", "guidance")))
+ space = goal_candidates(space, targets, tree, context["inputs"],
+ (observation.screenshot.width, observation.screenshot.height))
+ if self.visual:
+ space.target_heads["visual"] = self.visual
+ if (work, state) in self.grounded_states:
+ space.controls.pop("visual_fallback", None)
+ for operation in space.target_heads:
+ space.target_heads[operation] = [c for c in space.target_heads[operation]
+ if _action_key(c, targets) not in blocked]
+ space.controls = {k: c for k, c in space.controls.items() if _action_key(c, targets) not in blocked}
+ context.update(self.interaction.finalize(space, targets, stage, self.memory, tree,
+ observation_ref=f"{stage['id']}:O{observation.step_idx}"))
+ control = self.interaction.control
+ if self.recovery_stage != (stage["id"], json.loads(control["scope_key"])[2]):
+ self.recovery_stage = (stage["id"], json.loads(control["scope_key"])[2])
+ self.recovery_calls, self.recovery_seconds = 0, 0.0
+ context["grounding_exhausted"] = (
+ self.recovery_calls >= session.config.goal_recovery_max_calls
+ or self.recovery_seconds >= session.config.goal_recovery_max_seconds
+ or control.get("ground_calls", 0) >= session.config.goal_recovery_max_calls
+ or control.get("ground_seconds", 0) >= session.config.goal_recovery_max_seconds)
+ if context["grounding_exhausted"]:
+ space.controls.pop("visual_fallback", None)
+ self.memory.block("capability_exhausted", control["scope_key"],
+ "No further visual grounding in this unchanged input scope; "
+ "use offered observed actions or return for another route.",
+ f"{stage['id']}:O{observation.step_idx}")
+ context["active_blocker"] = self.memory.active_blocker
+ stage["interaction"] = self.interaction.snapshot()
+ missing = context["unbound_inputs"] and not any(c.source == "goal_input"
+ or c.metadata.get("commit_action") for c in space.candidates())
+ need_grounding = ((semantic_missing or self.no_effect >= 2 or cycle or missing)
+ and not self.visual and not context["grounding_exhausted"]
+ and (work, state) not in self.grounded_states)
+ if need_grounding and session.components.grounder is not None:
+ if not self._ground(session, observation, screenshot, context, state):
+ return self._finish(session, observation)
+ if self.visual:
+ raise ObservationRequired("local_visual_recovery_returned")
+ if missing and not self.visual and stage["actions"] >= 4:
+ self.end_reason = "capability_missing: " + ", ".join(context["unbound_inputs"])
+ return self._finish(session, observation)
+ if context["conflicting_fields"] and not any(c.source == "goal_input" for c in space.candidates()):
+ self.end_reason = "stage_input_conflict: " + ", ".join(context["conflicting_fields"])
+ return self._finish(session, observation)
+ if (stage["purpose"] == "produce" and stage["actions"] >= 4
+ and context["missing_required_fields"] and not context["next_intent"]):
+ self.end_reason = "stage_inputs_missing: " + ", ".join(context["missing_required_fields"])
+ return self._finish(session, observation)
+ choice_context = self.interaction.choice_context(context)
+ current_goal = choice_context["objective"]
+ ranking = session.components.selector.rank(
+ stage_goal=current_goal, intent=json.dumps(choice_context, ensure_ascii=False),
+ active_window=ET.fromstring(tree).get("foreground_title", "") if tree else "",
+ targets=targets, action_space=space,
+ recent_actions=session.recent[-8:], screenshot=screenshot)
+ winner = ranking.winner
+ operation = winner.operation if winner else "return_to_planner"
+ target = next((t for t in targets if winner and t.id == winner.target_id), None)
+ action = session.components.compile_gui(winner, target) if winner else None
+ session.events.append({"route": "gui", "step": observation.step_idx, "mode": "goal",
+ "selector_backend": session.config.fast_backend,
+ "stage_id": self.memory.active_stage["id"],
+ "plan": {"stage_goal": context["goal"], "current_goal": current_goal},
+ "input_frontier": {k: context[k] for k in (
+ "next_intent", "missing_required_fields", "unbound_inputs",
+ "conflicting_fields", "satisfied_inputs")},
+ "recovery_scope": {
+ k: context[k] for k in ("blocking_overlay", "overlay_bounds",
+ "document_recovery", "application_recovery")
+ if k in context},
+ "candidates": [c.compact() for c in space.candidates()],
+ "winner": winner.compact() if winner else None,
+ "avoid_actions": context["avoid_actions"],
+ "execution_control": context["execution_control"],
+ "ranking": ranking_dict(ranking),
+ "action": action.to_dict() if action else None})
+ if operation == "visual_fallback" and session.components.grounder is not None:
+ if (work, state) not in self.grounded_states:
+ if not self._ground(session, observation, screenshot, context, state):
+ return self._finish(session, observation)
+ if self.visual:
+ raise ObservationRequired("visual_grounder_returned")
+ if action is None:
+ self.end_reason = "stage_done_requested" if operation == "done" else "returned_to_planner"
+ return self._finish(session, observation)
+ self.pending = {"kind": action.kind, "operation": operation,
+ "target": winner.description, "action": action.to_dict(),
+ "state": _state_key(targets), "loop_state": state,
+ "candidate_key": _action_key(winner, targets),
+ "before_screenshot": observation.screenshot.path,
+ "expected_text": winner.metadata.get("text"),
+ "input_mode": winner.metadata.get("input_mode"),
+ "target_semantic": target.semantic_key if target else None,
+ "field_label": winner.metadata.get("field_label"),
+ "navigation_key": _navigation_key(winner, target, work, state),
+ "visual_action": operation == "scroll" or winner.source == "visual_grounder"
+ or target is not None and target.role == "canvas",
+ "fraction": winner.metadata.get("fraction")}
+ self.pending["overlay_bounds"] = context.get("overlay_bounds")
+ self.pending.update(control_scope=context["execution_control"]["scope_key"],
+ control_state=self.interaction.content_state)
+ self.pending.update({k: winner.metadata.get(k) for k in (
+ "interaction_binding", "commit_action", "key_sequence", "execution_family")})
+ self.pending["effect_bounds"] = winner.metadata.get("visual_bounds") or (
+ [target.x, target.y, target.width, target.height] if target else None)
+ return action
+
+ def _ground(self, session, observation, screenshot, context, state):
+ control = self.interaction.control
+ if (self.recovery_calls >= session.config.goal_recovery_max_calls
+ or self.recovery_seconds >= session.config.goal_recovery_max_seconds
+ or control is not None and (control.get("ground_calls", 0)
+ >= session.config.goal_recovery_max_calls or control.get("ground_seconds", 0)
+ >= session.config.goal_recovery_max_seconds)):
+ self.end_reason = "capability_missing: visual grounding budget exhausted"
+ self.memory.block("capability_exhausted", control["scope_key"], self.end_reason,
+ f"{self.memory.active_stage['id']}:O{observation.step_idx}")
+ session.events.append({"route": "recovery_exhausted", "step": observation.step_idx,
+ "stage_id": self.memory.active_stage["id"],
+ "calls": self.recovery_calls, "seconds": round(self.recovery_seconds, 3)})
+ return False
+ started = time.monotonic()
+ self.recovery_calls += 1
+ self.grounded_states[(self.memory.active_stage["work_item_id"], state)] = (
+ str(screenshot), self.memory.active_stage["id"])
+ plan = {**self.interaction.choice_context(context),
+ "current_goal": context.get("next_intent") or context["completion_criteria"],
+ "stage_goal": context["completion_criteria"],
+ "action_proposals": [{"operation": "type_text", **v} for v in context["inputs"]
+ if not launcher_input(v) or state[0] == "GNOME Overview"]}
+ self.visual = session.components.grounder(session.instruction, plan, screenshot)
+ elapsed = time.monotonic() - started
+ self.recovery_seconds += elapsed
+ if control is not None:
+ control.update(ground_calls=control.get("ground_calls", 0) + 1,
+ ground_seconds=control.get("ground_seconds", 0.0) + elapsed)
+ self.visual_frame = {"step": observation.step_idx, "state": state, "screenshot": str(screenshot)}
+ # The new frame was located after the invalidating action, not before it.
+ self.interaction.invalidate_visual = False
+ session.events.append({"route": "recovery", "step": observation.step_idx,
+ "stage_id": self.memory.active_stage["id"],
+ "method": "visual_grounding", "candidate_count": len(self.visual),
+ "avoid_actions": context.get("avoid_actions", []),
+ "attempt": self.recovery_calls})
+ return True
diff --git a/src/tandem/runtime/session.py b/src/tandem/runtime/session.py
index 460337c..63c796e 100644
--- a/src/tandem/runtime/session.py
+++ b/src/tandem/runtime/session.py
@@ -18,6 +18,7 @@
from ..models.ranking import ranking_dict
from ..protocol.models import Action, CliAction, FailAction, GuiAction, Observation, WaitAction
from .episode import handle_done, resolve_done_probe, verify_pending
+from .goal_stage import GoalExecution
from .stages import StageExecution, recovery_scopes
from .stopping import NO_ACTION_LIMIT, budget_exhausted
@@ -30,6 +31,7 @@ def __init__(self, components: Components, *, config: SessionConfig | None = Non
self.begin(requirements=[]) # 构造即重置,但不触发 requirements 抽取
def begin(self, instruction: str = "", requirements: list | None = None) -> None:
+ self.goal = None
self.instruction = instruction
# requirements=None(真实 run)时走组件抽取(旧 runtime 同款);
# 显式 [](mock 链路)跳过 LLM 调用
@@ -45,9 +47,14 @@ def begin(self, instruction: str = "", requirements: list | None = None) -> None
self.stage = (
StageExecution(self.config.stage_max_actions) if self.config.execution_mode == "stage" else None
)
+ if self.config.execution_mode == "goal":
+ self.goal = GoalExecution(self.components.memory_factory(instruction, requirements),
+ self.config.stage_max_actions, self.components.interaction_factory())
# ------------------------------------------------------------------ 主入口
def act(self, observation: Observation) -> Action:
+ if self.goal is not None:
+ return self.goal.act(self, observation)
tree = self._read_tree(observation)
if self.pending is not None:
verify_pending(self, observation, tree)
diff --git a/src/tandem/verification/goal_verifier.py b/src/tandem/verification/goal_verifier.py
new file mode 100644
index 0000000..ad1171e
--- /dev/null
+++ b/src/tandem/verification/goal_verifier.py
@@ -0,0 +1,423 @@
+"""One requirement-bound semantic verification per ended goal stage, never per GUI action."""
+
+import base64
+import json
+from copy import deepcopy
+from pathlib import Path
+
+from ..models.slow import ModelResponseError
+
+PROMPT = """Independently verify the ENDED desktop subtask against its fixed work unit and ORIGINAL
+requirements. Do not judge each intermediate action, demand later pages earlier, or trust the actor's
+printed success. Return only JSON:
+{stage_outcome:"met|not_met|uncertain",unit_met:false,delivery_state:"none|implemented|saved|verified",
+summary:"specific observed result",remaining:["concrete missing part"],evidence_refs:["supplied id"],
+facts:[{key:"registered or concrete reusable key",value:"exact content including numbers/units",
+status:"confirmed|uncertain",scope:"task|stage",source_ref:"actual file/page/location",
+evidence_refs:["supplied id"],complete:true}],
+materials:[{source_ref:"actual file/url",covered_parts:["page/section"],
+summary:"reusable source content",evidence_refs:["supplied id"]}],
+requirement_checks:[{id:"R1",status:"met|not_met|uncertain",evidence_refs:["supplied id"]}],
+artifacts:[{target_path:"actual final path",working_path:"actual working path",save_state:"modified|saved",
+evidence_refs:["supplied id"]}],invalidated_work_item_ids:[],
+milestones:[{key:"stable target/section identifier",summary:"actual delivered increment",
+evidence_refs:["current supplied id"]}],policy_violation:"",
+progress_kind:"none|navigation|source|content|save|inspection",observed_errors:[],
+unit_assessment:{work_item_id:"exact fixed id",status:"met|not_met|uncertain",
+evidence_refs:[],remaining:[]}}.
+For purpose=prepare, OMIT unit_assessment: the host records not_assessed without asking you to judge
+the delivery unit. Never accept a delivery unit during preparation.
+Use [] when no update is established. Reference only the supplied evidence IDs. Preserve original
+requirements and source facts. Extract essential facts NOW from real observed material/CLI output so
+later stages do not reread it; unobserved contents remain unknown. Preparation uses delivery_state=none.
+Use the exact facts_needed key when observed information answers that key. Keep partial source coverage
+in materials even if not all required facts have been acquired. Opening a reference image, a contact
+sheet, an audit or a script
+that reports failure is not delivery progress. A finished stage need not satisfy a whole requirement.
+unit_met=true ONLY when the full fixed work unit is satisfied; for delivery it requires verified level
+and concrete evidence. Saved requires actual persistence to the target, not a working copy or title.
+File existence/compilation does not prove content or function. Do not invent a demand to repeat every
+successful intermediate action. Mark unmet/uncertain criteria explicitly and preserve actual progress.
+Later contradictions invalidate affected units; do not invalidate old work from an uncertain screenshot
+alone. Do not read evaluators, hidden grading rules, answers or rewards. Be concise and evidence-based."""
+
+PROMPT += """
+Check protected state explicitly: opening a reference must not destroy an unsaved task form/document.
+Do not claim another tab, preserved fields or a prior save still exists without current supporting evidence.
+When current evidence proves a form reset, edits lost or the wrong artifact replaced the working result,
+return the affected delivery work_item_id in invalidated_work_item_ids, even in a prepare stage.
+Keep stable source facts, but withdraw stale UI/delivery milestones. A temporary working copy is not the
+user's final deliverable: artifacts.target_path names that deliverable, working_path names the actual copy.
+When publishing/reopening a previous working copy at its final path, include replaces_working_path with
+the previous copy's exact path so its pending publication can be retired.
+For file opening, inspect the resulting document instead of demanding its now-closed path entry.
+A pending Open/Import confirmation is not a completed file handoff, unless a preview alone was requested.
+"""
+
+PROMPT += '''
+delivery_state describes ONLY the CURRENT stage's new delivery, never an artifact's historical level.
+For navigation/source/no new work use delivery_state=none; the host retains earlier saved/verified state.
+Registered source-key updates belong to purpose=prepare acquisition stages, not form-state snapshots
+during editing. Keep current form values in stage-scope field-specific notes, never overwrite a source key.
+scope=task means reusable SOURCE CONTENT, not whole-task completion. scope=stage is only transient UI
+state. A confirmed paragraph or data field is task-scope even while the rest of its source is unread.
+Not seeing a source again does not contradict its recorded facts. Retain confirmed source values;
+change one only with actual contradictory evidence and a nonempty correction_reason on that fact.
+Use the precise registered source key when it is fully answered; store partial sections as separate
+reusable facts/materials. Do not mark an entire packet confirmed when required fields are missing.
+remaining must describe THIS stage's unmet exit conditions, not a list of every later task obligation.
+Only mark unit_met when the original full unit is satisfied. Historical registered evidence may support
+source facts; stage completion must also cite at least one CURRENT evidence ID.
+Use the request's response_template for the mandatory keys and exact work_item_id; populate it with
+observed evidence, not empty defaults. facts[].value is ALWAYS a JSON STRING, including numbers, dates,
+lists and tables: use "25", not 25, and readable text, not an object or array. Optional facts[].entries
+maps string keys to STRING values; store numbers as strings there too. A stage purpose=prepare is
+source acquisition only even when attached to a delivery unit; it cannot finish that delivery unit.
+For lists/tables/options, include entries:{"stable item/row/position":"observed value"} and complete:false
+until the requested coverage is actually observed. New entries extend the old collection; conflicting
+entries or scalar changes require current evidence and correction_reason. Never label a partial list
+as a complete registered dependency. Temporary UI state is not required source content.
+This request contains selected evidence, not the whole execution log. output_omitted_chars marks an
+explicit excerpt: do not claim to have checked omitted contents or a complete source packet. Extract
+only supported fields; ask for the missing section rather than another full-document audit.
+For prepare, answer fact_targets using their EXACT registered keys and record partial coverage;
+unit_met=false, delivery_state=none, and do not assess unrelated delivery requirements.
+For produce/save/verify, check only the current stage exit condition and its fixed delivery unit.
+Record partial implementation/saving without demanding all future task units. Return only memory deltas,
+not a rewrite of existing facts/materials, and no new optional research or inspection requirements.
+Missing source fields do not invalidate an independently useful, correctly implemented partial stage.
+If that bounded exit condition is met, report stage_outcome=met with unit_met=false and the observed
+implemented/saved level. Keep the full unit incomplete and preserve its missing values for later work.
+Incomplete prose facts may be appended across stages: complete:false preserves earlier sections without
+claiming a complete source. Mark complete:true only when the exact registered field/section is fully answered.
+Never replace a known inventory with the contents of a different document under the same scalar key;
+prefer document/field-specific facts and preserve source_ref. Do not guess identity or ownership.
+For a real delivered increment, record a milestone with a stable artifact/section/field-group key.
+Reuse the same key from fixed_work_item.milestones for the same increment; never key by stage ID,
+rewording, research, audit files, navigation, or opening a source. New source facts are not delivery.
+milestones:[] when nothing was actually implemented, saved or independently checked.
+CLI actor assertions, ZIP/XML validity and marker strings establish neither visual fidelity nor working
+animations. They can support saving only. Current GUI inspection must demonstrate visual requirements;
+a blocking dialog is not evidence that the underlying content was checked. Do not weaken failed checks.
+Respect ORIGINAL interaction restrictions. Filling website fields is not permission to edit the web
+application's source, inject reload scripts, add missing options or remove validation. If the actions
+violate these boundaries, return policy_violation with the specific observed action and do not accept it.
+Planner method/guidance and invented workflow preferences are NOT user policy. Wrong application,
+wrong shortcut, wrong navigation and incomplete work are recoverable stage errors, not policy violations.
+For a genuine violation also return policy_constraint and policy_evidence_refs. policy_constraint must
+be an EXACT quote of an ORIGINAL explicit prohibition, or one of website_source_tampering,
+evaluator_access, credential_access for the host's integrity boundaries. policy_evidence_refs must cite
+CURRENT actions demonstrating the violation, not an old screenshot or the planner's proposed method.
+Do not infer tampering from ordinary CLI editing of a requested output document. Opening an unintended
+application is not a prohibition violation unless the user explicitly forbids opening it.
+Keep the existing working_path for the same target_path. When current evidence demonstrates a real
+working-copy migration preserving prior increments, include replaces_working_path with the old exact
+working path on that artifact. A fresh source copy or unrelated per-section file is not such a migration.
+progress_kind and observed_errors are mandatory. For produce/save/verify also return unit_assessment
+with the exact fixed work_item_id, status met|not_met|uncertain, evidence_refs and remaining arrays.
+stage_outcome and remaining judge ONLY stage_acceptance. unit_assessment judges ONLY unit_acceptance,
+NOT all sibling units or the overall requirement. The entire task is NOT being accepted here.
+If a save/open unit is fully supported, mark that UNIT met even when sibling animation/content units
+remain pending. Do not demand their contents in this unit. If unit_assessment.status=met, return
+unit_met=true and delivery_state=verified with current evidence. Preparation omits unit_assessment.
+Opening/maximizing windows, menus, scrolling, and dismissing dialogs are navigation: delivery_state=none,
+unit_met=false,milestones:[], even if their bounded preparation stage succeeded. Source reading is source,
+not content delivery. A saved partial artifact may be save progress without finishing its whole unit.
+observed_errors is [{message:"EXACT error visible in an action receipt or current screenshot",
+application:"observed application",command_name:"exact command if explicitly named, otherwise empty",
+evidence_refs:["CURRENT id"]}]. Do not replace the error with remaining steps or invent errors.
+Unknown command means the command was not recognized; extra arguments are not evidence of support.
+Corrective advice belongs in remaining, never in observed_errors or confirmed source facts.
+'''
+
+
+def result_error(value):
+ """Precise contract failures; never invent evidence or coerce unsupported results."""
+ for key, choices in {"stage_outcome": {"met", "not_met", "uncertain"},
+ "delivery_state": {"none", "implemented", "saved", "verified"}}.items():
+ if value.get(key) not in tuple(choices):
+ return key + " must be one of " + str(sorted(choices))
+ for key, kind in {"unit_met": bool, "summary": str}.items():
+ if type(value.get(key)) is not kind:
+ return key + " must be " + kind.__name__
+ lists = ("remaining", "evidence_refs", "facts", "materials", "requirement_checks", "artifacts",
+ "invalidated_work_item_ids")
+ for key in lists:
+ if not isinstance(value.get(key), list):
+ return key + " must be an array (use [] for no updates)"
+ for key in ("remaining", "evidence_refs", "invalidated_work_item_ids"):
+ if any(not isinstance(x, str) for x in value[key]):
+ return key + " must contain only strings"
+ if value.get("progress_kind", "none") not in {
+ "none", "navigation", "source", "content", "save", "inspection"}:
+ return "progress_kind must describe observed navigation, source, content, save or inspection"
+ if not isinstance(value.get("observed_errors", []), list):
+ return "observed_errors must be an array"
+ for error in value.get("observed_errors", []):
+ if (not isinstance(error, dict) or any(not isinstance(error.get(k), str) for k in (
+ "message", "application", "command_name")) or not error["message"].strip()
+ or not isinstance(error.get("evidence_refs"), list)
+ or any(not isinstance(ref, str) for ref in error["evidence_refs"])):
+ return "observed_errors need exact message, application, command_name and evidence_refs"
+ check = value.get("unit_assessment")
+ if "unit_assessment" in value and (not isinstance(check, dict)
+ or not isinstance(check.get("work_item_id"), str)
+ or check.get("status") not in {"met", "not_met", "uncertain", "not_assessed"}
+ or any(not isinstance(check.get(k), list) or any(not isinstance(x, str) for x in check[k])
+ for k in ("remaining", "evidence_refs"))):
+ return "unit_assessment needs work_item_id, status, remaining and evidence_refs"
+ if not isinstance(value.get("policy_violation", ""), str):
+ return "policy_violation must be a string"
+ if not isinstance(value.get("policy_constraint", ""), str):
+ return "policy_constraint must be a string"
+ if (not isinstance(value.get("policy_evidence_refs", []), list)
+ or any(not isinstance(ref, str) for ref in value.get("policy_evidence_refs", []))):
+ return "policy_evidence_refs must be an array of strings"
+ if not isinstance(value.get("milestones", []), list):
+ return "milestones must be an array"
+ for record in value.get("milestones", []):
+ if (not isinstance(record, dict) or any(not isinstance(record.get(k), str) or not record[k].strip()
+ for k in ("key", "summary"))
+ or not isinstance(record.get("evidence_refs"), list)
+ or any(not isinstance(ref, str) for ref in record["evidence_refs"])):
+ return "milestones need a stable key, summary and evidence_refs array of strings"
+ fields = {"facts": ("key", "value", "source_ref"), "materials": ("source_ref", "summary"),
+ "requirement_checks": ("id",), "artifacts": ("target_path", "working_path")}
+ enums = {"facts": {"status": {"confirmed", "uncertain"}, "scope": {"task", "stage"}},
+ "requirement_checks": {"status": {"met", "not_met", "uncertain"}},
+ "artifacts": {"save_state": {"modified", "saved"}}}
+ for collection, strings in fields.items():
+ for index, record in enumerate(value[collection]):
+ path = f"{collection}[{index}]"
+ if not isinstance(record, dict):
+ return path + " must be an object"
+ for key in strings:
+ if not isinstance(record.get(key), str):
+ return path + "." + key + " must be a string"
+ arrays = ("evidence_refs", "covered_parts") if collection == "materials" else ("evidence_refs",)
+ for key in arrays:
+ if not isinstance(record.get(key), list) or any(not isinstance(x, str) for x in record[key]):
+ return path + "." + key + " must be an array of strings"
+ for key, choices in enums.get(collection, {}).items():
+ if record.get(key) not in tuple(choices):
+ return path + "." + key + " must be one of " + str(sorted(choices))
+ if collection == "facts":
+ if "complete" in record and type(record["complete"]) is not bool:
+ return path + ".complete must be a boolean"
+ if "entries" in record and (not isinstance(record["entries"], dict) or any(
+ not isinstance(k, str) or not isinstance(v, str)
+ for k, v in record["entries"].items())):
+ return path + ".entries must map strings to strings"
+ return ""
+
+
+def valid_result(value):
+ return not result_error(value)
+
+
+def _excerpt(text, limit):
+ if len(text) <= limit:
+ return text
+ return text[:limit // 2] + "\n[... omitted source content; not inspected ...]\n" + text[-limit // 2:]
+
+
+def _actions(history, output_budget):
+ """Keep receipts and semantic deltas, not repeated snapshot IDs/ancestry/whole trees."""
+ actions = []
+ previous = {}
+ output_records = max(1, sum(bool(r.get("full_output_ref") or r.get("stdout") or r.get("stderr"))
+ for r in history))
+ limit = max(1000, output_budget // output_records)
+ for record in history:
+ action = {k: record[k] for k in ("evidence_ref", "kind", "operation", "target", "returncode",
+ "observed_change", "relevant_change", "target_effect",
+ "full_output_ref") if k in record}
+ if record.get("kind") != "cli" and record.get("action"):
+ action["action"] = record["action"]
+ output = dict(record)
+ if record.get("full_output_ref"):
+ output.update(json.loads(Path(record["full_output_ref"]).read_text(encoding="utf-8")))
+ omitted = {}
+ for key in ("stdout", "stderr"):
+ text = output.get(key, "")
+ if text:
+ action[key] = _excerpt(text, limit)
+ if len(text) > limit:
+ omitted[key] = len(text) - limit
+ if omitted:
+ action["output_omitted_chars"] = omitted
+ changes, current = [], {}
+ for control in record.get("observed_controls", []):
+ key = (control.get("role"), control.get("name"), control.get("x"), control.get("y"))
+ value = {k: control[k] for k in ("role", "name", "value", "focused", "checked", "selected")
+ if k in control}
+ current[key] = value
+ if value != previous.get(key):
+ changes.append(value)
+ previous = current
+ if changes:
+ action["control_changes"] = changes
+ actions.append(action)
+ return actions
+
+
+def _failure_result(memory, history, end_reason, evidence, error=None):
+ """No semantic success claim from a known error or an entirely ineffective local attempt."""
+ pure_no_effect = (history and all(h.get("kind") == "gui"
+ and not h.get("relevant_change", h.get("observed_change"))
+ for h in history))
+ skip = (error is not None or end_reason == "execution_error" or end_reason.startswith((
+ "stage_inputs_missing:", "stage_input_conflict:", "capability_missing:"))
+ or not history and end_reason != "stage_done_requested"
+ or pure_no_effect and not any(h.get("operation") == "type_text" for h in history)
+ and memory.active_stage["purpose"] not in {"save", "verify"})
+ if not skip:
+ return None
+ error = error or next((h.get("stderr") or h.get("stdout") for h in reversed(history)
+ if h.get("returncode")), "")
+ return {"stage_outcome": "uncertain", "unit_met": False, "delivery_state": "none",
+ "summary": "No semantic acceptance: " + end_reason
+ + (". Observed error: " + error[-1200:] if error else ""),
+ "remaining": [end_reason + ": inspect partial effects and change method; do not repeat it."],
+ "evidence_refs": list(evidence)[-1:], "facts": [], "materials": [],
+ "requirement_checks": [], "artifacts": [], "invalidated_work_item_ids": [],
+ "check_mode": "local_failure"}
+
+
+def verify_goal(client, instruction, screenshot, *, memory, history, end_reason, evidence):
+ local = _failure_result(memory, history, end_reason, evidence)
+ if local is not None:
+ local["failure_kind"] = end_reason
+ return local
+ stage = memory.active_stage
+ work = memory.work_items[stage["work_item_id"]]
+ targets = stage.get("fact_targets", work["facts_needed"])
+ keys = set(stage["input_fact_keys"]) | {f["key"] for f in targets}
+ budget = client.config.stage_verify_max_chars
+ actions = _actions(history, budget // 2)
+ preparing = stage["purpose"] == "prepare"
+ template = {"stage_outcome": "uncertain", "unit_met": False, "delivery_state": "none",
+ "summary": "Describe current observed evidence", "remaining": [], "evidence_refs": [],
+ "facts": [], "materials": [], "requirement_checks": [], "artifacts": [],
+ "invalidated_work_item_ids": [], "milestones": [], "policy_violation": "",
+ "progress_kind": "none", "observed_errors": []}
+ if not preparing:
+ template["unit_assessment"] = {"work_item_id": work["id"], "status": "uncertain",
+ "evidence_refs": [], "remaining": []}
+ context = {"task": instruction,
+ "response_template": template,
+ "stage_acceptance": {"criterion": stage["completion_criteria"], "scope": "ended_stage_only"},
+ "unit_acceptance": ({"work_item_id": work["id"], "scope": "not_assessed_during_prepare"}
+ if preparing else {"work_item_id": work["id"],
+ "criterion": work["completion_criteria"],
+ "scope": "this_fixed_unit_only_not_all_original_requirements"}),
+ "sibling_units_not_current_acceptance": [{"id": w["id"], "goal": w["goal"]}
+ for w in memory.work_items.values() if w["kind"] == "deliver" and w["id"] != work["id"]
+ and w["requirement_ids"] == work["requirement_ids"]],
+ "stage": {k: v for k, v in stage.items() if k != "command"},
+ "command_excerpt": _excerpt(stage.get("command", ""), 3000),
+ "fixed_work_item": {**work, "facts_needed": targets},
+ "requirements": [{k: r[k] for k in ("id", "requirement", "kind")}
+ for r in memory.requirements if r["id"] in work["requirement_ids"]
+ or r["kind"] == "constraint"],
+ "known_facts": [f for f in memory.facts.values() if f["key"] in keys],
+ "end_reason": end_reason, "actual_actions": actions, "evidence_ids": list(evidence),
+ "historical_evidence_ids": sorted(memory.known_evidence)}
+ text = json.dumps(context, ensure_ascii=False)
+ if len(text) > budget:
+ # Receipt deltas stay intact. Explicitly excerpt source text, never silently claim full coverage.
+ for index, original in enumerate(context["known_facts"]):
+ fact = {k: v for k, v in original.items() if k != "entries"}
+ fact["value"] = _excerpt(fact["value"], max(1000, budget // (2 * max(1, len(keys)))))
+ if fact["value"] != original["value"]:
+ fact["value_excerpt_only"] = True
+ context["known_facts"][index] = fact
+ text = json.dumps(context, ensure_ascii=False)
+ content = [{"type": "text", "text": text}]
+ source_frames = [h["screenshot"] for h in history if h.get("source_frame")]
+ paths = source_frames[-5:] if stage["purpose"] == "prepare" else (
+ [history[0]["screenshot"]] if history else [])
+ frames = [Path(p) for p in dict.fromkeys(paths + [str(screenshot)])]
+ for path in frames:
+ content.extend([{"type": "text", "text": "Observed screenshot: " + str(path)},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,"
+ + base64.b64encode(path.read_bytes()).decode("ascii")}}])
+ messages = [{"role": "system", "content": PROMPT}, {"role": "user", "content": content}]
+
+ def validate(value):
+ error = result_error(value)
+ if not error:
+ check = value.get("unit_assessment", {})
+ required = ("progress_kind", "observed_errors") + (() if preparing else ("unit_assessment",))
+ if not all(key in value for key in required):
+ error = "include mandatory fields: " + ", ".join(required)
+ elif preparing and (value["unit_met"] or value["delivery_state"] != "none"
+ or value.get("milestones")):
+ error = "prepare cannot accept a delivery unit or claim delivery progress"
+ elif not preparing and check.get("work_item_id") != work["id"]:
+ error = "unit_assessment.work_item_id must be " + work["id"]
+ elif not preparing and (check.get("status") == "met") != value["unit_met"]:
+ error = "unit_met must agree with this unit_assessment, not whole-task completion"
+ elif value["unit_met"] and value["delivery_state"] != "verified":
+ error = "a met delivery unit requires verified evidence for its own fixed criterion"
+ elif value["progress_kind"] in {"navigation", "source", "none"} and (
+ value["unit_met"] or value["delivery_state"] != "none" or value.get("milestones")):
+ if value["unit_met"] or value.get("milestones"):
+ error = "navigation/source/no delivery cannot claim units or delivery milestones"
+ else:
+ value["historical_delivery_claim"] = value["delivery_state"]
+ value.update(delivery_state="none", artifacts=[])
+ if not error:
+ allowed = set(evidence) | memory.known_evidence
+ records = [value, *value["facts"], *value["materials"],
+ *value["artifacts"], *value["requirement_checks"], *value.get("milestones", []),
+ *value.get("observed_errors", [])]
+ if not preparing:
+ records.append(value["unit_assessment"])
+ unknown = sorted({ref for r in records for ref in r["evidence_refs"] if ref not in allowed})
+ if unknown:
+ error = "evidence_refs contain unregistered IDs: " + json.dumps(unknown)
+ elif value["stage_outcome"] == "met" and not set(value["evidence_refs"]) & set(evidence):
+ error = "a met stage must cite at least one CURRENT evidence ID"
+ elif any(not set(item["evidence_refs"]) or not set(item["evidence_refs"]) <= set(evidence)
+ for item in value["observed_errors"]):
+ error = "observed_errors must cite CURRENT observed evidence"
+ elif value["unit_met"] and not set(value["unit_assessment"]["evidence_refs"]) & set(evidence):
+ error = "a met unit must cite CURRENT evidence for its fixed criterion"
+ if error:
+ messages.append({"role": "user", "content":
+ "Invalid stage verification JSON: " + error + ". Return the COMPLETE corrected object "
+ "with every mandatory key, without changing the observed facts or inventing evidence."})
+ return not error
+
+ try:
+ result = client.json_request(messages, validate, role="verify_stage")
+ except ModelResponseError as error:
+ result = _failure_result(memory, history, "verification_output_invalid", evidence, str(error))
+ return {**result, "failure_kind": "verification_output_invalid"}
+ result = deepcopy(result)
+ if preparing:
+ result["unit_assessment"] = {"work_item_id": work["id"], "status": "not_assessed",
+ "evidence_refs": [], "remaining": []}
+ if result.get("progress_kind") in {"navigation", "source", "none"}:
+ result.update(delivery_state="none", unit_met=False, milestones=[])
+ if stage["route"] == "cli" and stage["purpose"] in {"produce", "save"}:
+ if result["delivery_state"] == "verified":
+ result.update(unit_met=False, delivery_state="saved",
+ acceptance_limited="cli_save_not_gui_acceptance")
+ if result.get("policy_violation"):
+ constraint = result.get("policy_constraint", "").strip()
+ refs = set(result.get("policy_evidence_refs", []))
+ current_actions = {h["evidence_ref"] for h in history if h.get("evidence_ref") in evidence}
+ original = (len(constraint) >= 12 and constraint in instruction and any(
+ token in constraint.casefold() for token in ("not ", "never ", "forbid", "禁止", "不得")))
+ integrity = constraint in {"website_source_tampering", "evaluator_access", "credential_access"}
+ fatal = bool(refs and refs <= current_actions and (original or integrity))
+ result.update(stage_outcome="not_met", unit_met=False, delivery_state="none", milestones=[],
+ artifacts=[], requirement_checks=[], remaining=[result["policy_violation"]],
+ failure_kind="policy_violation" if fatal else "recoverable_stage_error")
+ if fatal:
+ result.update(facts=[], materials=[])
+ result.update(check_mode="source" if stage["purpose"] == "prepare" else "delivery",
+ context_chars=len(text), image_count=len(frames))
+ return result
diff --git a/src/tandem/verification/verifier.py b/src/tandem/verification/verifier.py
index 18c817d..3e44ba6 100644
--- a/src/tandem/verification/verifier.py
+++ b/src/tandem/verification/verifier.py
@@ -37,6 +37,7 @@ def verify_observation(
requirements=None,
output_files=None,
execution_result=None,
+ task_memory=None,
):
context = {
"task": instruction,
@@ -50,6 +51,8 @@ def verify_observation(
}
if execution_result is not None:
context["execution_result"] = execution_result
+ if task_memory is not None:
+ context["verified_task_memory"] = task_memory
content = [{"type": "text", "text": json.dumps(context, ensure_ascii=False)}]
for label, path in (("BEFORE", previous), ("CURRENT", screenshot)):
if path is not None:
@@ -107,6 +110,13 @@ def verify_observation(
"A CLI read-only inspection can be achieved without a GUI change; preserve state_changed=false "
"in that case. Report contradictory command failures instead of assuming the intended edit happened."
)
+ if task_memory is not None:
+ messages[0]["content"] += (
+ " For whole-task verification, use supplied independently verified stage evidence and "
+ "confirmed source facts as historical evidence; do not require every prior page or source "
+ "to appear simultaneously in the current screenshot. The ledger is not infallible: current "
+ "contradictions override it. A pending publication or unsaved working copy is not delivery."
+ )
result = client.json_request(
messages,
lambda value: (
@@ -147,6 +157,9 @@ def verify_observation(
not exists for exists in (output_files or {}).values()
):
result.update(task_status="incomplete", outcome="not_achieved")
+ if task_memory and any(a.get("delivery_pending") for a in task_memory["artifacts"]):
+ result.update(task_status="incomplete", outcome="not_achieved",
+ evidence=result["evidence"] + " Working-copy publication remains pending.")
return result
diff --git a/src/tandem_eval/__main__.py b/src/tandem_eval/__main__.py
index b4b30a8..8dad50e 100644
--- a/src/tandem_eval/__main__.py
+++ b/src/tandem_eval/__main__.py
@@ -36,7 +36,7 @@ def main(argv: list[str] | None = None) -> int:
run_cmd.add_argument("--allow-cli", choices=["true", "false"], default=None,
help="harness 是否允许 CLI 路由")
run_cmd.add_argument("--route-policy", choices=["auto", "gui", "cli"], default=None)
- run_cmd.add_argument("--execution-mode", choices=["step", "stage"], default=None)
+ run_cmd.add_argument("--execution-mode", choices=["step", "stage", "goal"], default=None)
run_cmd.add_argument("--stage-max-actions", type=int, default=None,
help="阶段内动作数上限,达到后请求剩余工作的重规划")
run_cmd.add_argument("--slow-model", default=None, metavar="NAME",
diff --git a/src/tandem_eval/browser.py b/src/tandem_eval/browser.py
index 358497c..4d8a41b 100644
--- a/src/tandem_eval/browser.py
+++ b/src/tandem_eval/browser.py
@@ -40,6 +40,8 @@
'gridcell','combobox','textbox','searchbox','spinbutton','slider']);
const selector = 'a[href],button,input,textarea,select,summary,[contenteditable="true"],' +
[...roles].map(role => '[role="' + role + '"]').join(',');
+ const clickable = element => element.tabIndex >= 0 || typeof element.onclick === 'function' ||
+ getComputedStyle(element).cursor === 'pointer';
const secretAutocomplete = new RegExp(
'(^|\\s)(current-password|new-password|one-time-code|cc-number|' +
'cc-csc|cc-exp|cc-exp-month|cc-exp-year)(\\s|$)');
@@ -55,9 +57,9 @@
if (['button','submit','reset','image'].includes(element.type)) return 'button';
if (element.type === 'search') return 'searchbox';
if (element.type === 'number') return 'spinbutton';
- if (['text','email','url','tel'].includes(element.type)) return 'textbox';
+ if (['text','email','url','tel','date'].includes(element.type)) return 'textbox';
}
- return null;
+ return clickable(element) ? 'button' : null;
};
const roleNames = {button:'push_button', link:'link', checkbox:'check_box', radio:'radio_button',
switch:'switch', tab:'page_tab', menuitem:'menu_item', option:'option', gridcell:'table_cell',
@@ -94,10 +96,27 @@
clone.querySelectorAll('input,textarea,select,[contenteditable="true"]').forEach(node => node.remove());
return norm(clone.innerText).slice(0, 240);
};
+ const itemContext = element => {
+ const row = element.closest('li,tr,[role="row"]');
+ if (row) return contextText(row);
+ for (let node = element.parentElement, depth = 0; node && depth < 4;
+ node = node.parentElement, depth++) {
+ if (node.matches('body,html,form,dialog,[role="dialog"]')) break;
+ const text = contextText(node);
+ if (text.length > name(element).length + 8 && text.length < 240 &&
+ node.querySelectorAll(selector).length <= 8) return text;
+ }
+ return contextText(element.closest('fieldset,form,dialog,[role="dialog"]'));
+ };
const elements = [];
let rawCount = 0;
- for (const element of document.querySelectorAll(selector)) {
+ for (const element of document.querySelectorAll(selector + ',div,span,li,td')) {
rawCount++;
+ if (!element.matches(selector)) {
+ if (!clickable(element)) continue;
+ const parent = element.parentElement;
+ if (parent && (parent.closest(selector) || clickable(parent))) continue;
+ }
if (!visible(element) || element.disabled || element.getAttribute('aria-disabled') === 'true') continue;
if (element.type === 'password' || element.type === 'hidden' || element.type === 'file' ||
secretAutocomplete.test(element.getAttribute('autocomplete') || '')) continue;
@@ -105,6 +124,8 @@
const cx = rect.left + rect.width / 2, cy = rect.top + rect.height / 2;
if (rect.width < 2 || rect.height < 2 || cx < 0 || cy < 0 ||
cx >= innerWidth || cy >= innerHeight) continue;
+ const hit = document.elementFromPoint(cx, cy);
+ if (!hit || !element.contains(hit)) continue; // A visible background node may still be occluded.
const domRole = roleOf(element);
if (!domRole) continue;
const hint = hintFor(rect);
@@ -112,17 +133,22 @@
const editable = !element.readOnly && element.getAttribute('aria-readonly') !== 'true' &&
(['textbox','searchbox','spinbutton'].includes(domRole) ||
(domRole === 'combobox' && ['INPUT','TEXTAREA'].includes(element.tagName)));
- const contextElement = element.closest('form,dialog,[role="dialog"],fieldset,li,tr,[role="row"]');
const item = {
index: elements.length + 1,
role: roleNames[domRole] || domRole,
name: label,
description: norm(element.getAttribute('aria-description') || element.getAttribute('title') ||
element.getAttribute('placeholder')),
- value: 'value' in element ? String(element.value).slice(0, 200) : '',
+ value: element.tagName === 'SELECT' ? norm(element.selectedOptions[0]?.label) :
+ 'value' in element ? String(element.value).slice(0, 200) : '',
+ input_type: ['INPUT','SELECT'].includes(element.tagName) ? element.type : '',
+ focused: document.activeElement === element,
+ date_order: element.type === 'date' ? new Intl.DateTimeFormat(navigator.language)
+ .formatToParts(new Date(2001,10,22)).filter(p => ['year','month','day'].includes(p.type))
+ .map(p => p.type) : [],
x: originX + rect.left, y: originY + rect.top, width: rect.width, height: rect.height,
scope: 'web_content', label_source: name(element) ? 'dom' : 'dom_nearby',
- nearby_text: contextText(contextElement),
+ nearby_text: itemContext(element),
operations: editable ? ['click','type_text'] : ['click'],
source: 'dom'
};
diff --git a/src/tandem_eval/config_schema.py b/src/tandem_eval/config_schema.py
index e6d9ceb..6d696c1 100644
--- a/src/tandem_eval/config_schema.py
+++ b/src/tandem_eval/config_schema.py
@@ -70,8 +70,8 @@
"harness.fast_model.image_stage.read",
)
-#: 快模型后端协议(决定用哪种排序传输:jev 的 /rank 还是 kev 的 /v1/systemone)
-FAST_BACKENDS = ("jev", "kev")
+#: 选择后端:jev /rank、kev /v1/systemone;planner 复用 slow_model,不需要独立 endpoint。
+FAST_BACKENDS = ("jev", "kev", "planner")
#: 慢模型每个 API 条目必填的键
SLOW_API_KEYS = ("base_url", "model")
@@ -175,7 +175,7 @@ def _validate_fast_models(config: dict) -> list[str]:
endpoint = api.get("endpoint")
endpoint_file = api.get("endpoint_file")
# endpoint 允许留空 = 自动从发现文件读取最新地址(服务闲时部署,地址会变)
- if _is_missing(endpoint) and _is_missing(endpoint_file):
+ if api.get("backend") != "planner" and _is_missing(endpoint) and _is_missing(endpoint_file):
problems.append(
f"harness.fast_model.apis.{name}.endpoint / .endpoint_file(至少给一个)"
)
@@ -241,8 +241,8 @@ def validate(config: dict) -> list[str]:
if not isinstance(execution, dict):
problems.append("harness.execution(应为对象)")
else:
- if execution.get("mode", "step") not in {"step", "stage"}:
- problems.append("harness.execution.mode(应为 step / stage)")
+ if execution.get("mode", "step") not in {"step", "stage", "goal"}:
+ problems.append("harness.execution.mode(应为 step / stage / goal)")
limit = execution.get("stage_max_actions", 24)
if type(limit) is not int or limit < 1:
problems.append("harness.execution.stage_max_actions(应为正整数)")
diff --git a/src/tandem_eval/entry.py b/src/tandem_eval/entry.py
index 7406ec7..e593304 100644
--- a/src/tandem_eval/entry.py
+++ b/src/tandem_eval/entry.py
@@ -103,8 +103,15 @@ def __init__(self, *, stage_mode=False):
self.stage_mode = stage_mode
def __call__(self, instruction, screenshot, targets, recent, tree,
- verified_focus, requirements, remaining_steps):
+ verified_focus=None, requirements=None, remaining_steps=None, *, memory=None,
+ remaining_seconds=None):
self.calls += 1
+ if memory is not None and self.calls == 1:
+ return {"done": False, "work_item_id": memory.ready_items()[0]["id"],
+ "route": "gui", "purpose": "produce", "goal": "mock click",
+ "completion_criteria": "mock receipt", "method": "mock click",
+ "inputs": [], "input_fact_keys": [], "guidance": "", "max_actions": 1,
+ "command": ""}
if self.calls == 1 and targets:
target = targets[0]
plan = {
@@ -146,19 +153,32 @@ def rank(self, **kwargs):
)
-def mock_components(*, stage_mode=False) -> Components:
+def mock_components(*, stage_mode=False, goal_mode=False) -> Components:
from tandem.gui.candidates import ActionCandidateBuilder
from tandem.gui.compile import gui_action
from tandem.gui.proposals import focused_candidates
from tandem.gui.semantics import enrich_targets
from tandem.gui.targets import A11yTargetBuilder
+ from tandem.memory.state import TaskMemory
+
+ def goal_verifier(*args, **kwargs):
+ return {"stage_outcome": "met", "unit_met": True, "delivery_state": "verified",
+ "summary": "mock evidence only", "remaining": [],
+ "evidence_refs": list(kwargs["evidence"]), "facts": [], "materials": [],
+ "requirement_checks": [], "artifacts": [], "invalidated_work_item_ids": []}
+
+ def requirements(instruction):
+ return [{"id": "R1", "requirement": instruction, "kind": "delivery", "units": [instruction],
+ "unit_fact_keys": [[]], "facts_needed": [], "output_path": None}] if goal_mode else []
return Components(
planner=_ScriptedPlanner(stage_mode=stage_mode), verifier=_ScriptedVerifier(),
- requirements=lambda instruction: [], selector=_FirstCandidateSelector(),
+ requirements=requirements, selector=_FirstCandidateSelector(),
target_builder=A11yTargetBuilder(), candidate_builder=ActionCandidateBuilder(),
semantic_enricher=enrich_targets,
focus_candidates=focused_candidates, grounder=None, compile_gui=gui_action,
+ memory_factory=TaskMemory if goal_mode else None,
+ stage_verifier=goal_verifier if goal_mode else None,
)
@@ -166,14 +186,16 @@ def build_session(spec: dict, *, llm_tap: LlmTap | None = None) -> TandemSession
config = SessionConfig(**spec.get("harness_config", {}))
benchmark = spec.get("benchmark")
if benchmark == "mock":
- return TandemSession(mock_components(stage_mode=config.execution_mode == "stage"), config=config)
+ return TandemSession(mock_components(stage_mode=config.execution_mode == "stage",
+ goal_mode=config.execution_mode == "goal"), config=config)
if benchmark in {"osworld1", "osworld2", "osworld21"}:
from tandem.components import build_default
from tandem.models.slow import SlowModelClient
secrets = spec["secrets"]
credentials = dict(secrets["slow_model"])
- endpoint = resolve_fast_model_endpoint(secrets.get("fast_model") or {})
+ endpoint = ("" if config.fast_backend == "planner" else
+ resolve_fast_model_endpoint(secrets.get("fast_model") or {}))
run_root = Path(spec["output_root"])
run_root.mkdir(parents=True, exist_ok=True)
client = SlowModelClient(credentials, config, run_root / "slow_model_http.jsonl",
diff --git a/src/tandem_eval/env.py b/src/tandem_eval/env.py
index 38f5c5f..77b2bd3 100644
--- a/src/tandem_eval/env.py
+++ b/src/tandem_eval/env.py
@@ -7,10 +7,13 @@
from __future__ import annotations
import hashlib
+import io
from dataclasses import dataclass, field
from pathlib import Path
from typing import Protocol
+from PIL import Image
+
from tandem.protocol.models import (
AccessibilityTree,
Action,
@@ -30,6 +33,8 @@ def reset(self, task: dict) -> Observation: ...
def execute(self, action: Action, *, budget: Budget, task: dict) -> tuple[Receipt, Observation]: ...
+ def observe(self, task: dict, *, budget: Budget, previous: Observation) -> Observation: ...
+
def fetch_task(self, task_id: str) -> dict: ...
def export_outputs(self, paths: list[str], episode_dir) -> list[dict]: ...
@@ -75,7 +80,9 @@ def fetch_task(self, task_id: str, domain: str = "") -> dict:
def _screenshot_ref(self, task_id: str, step_idx: int) -> tuple[str, str]:
key = f"{task_id}:{step_idx}"
if key not in self.shots:
- self.shots[key] = f"png-{key}".encode()
+ image = io.BytesIO()
+ Image.new("RGB", (1920, 1080), "white").save(image, format="PNG")
+ self.shots[key] = image.getvalue()
digest = hashlib.sha256(self.shots[key]).hexdigest()
if self.base_dir is not None:
target = self.base_dir / "screenshots" / f"step_{step_idx:02d}.png"
@@ -140,6 +147,9 @@ def execute(self, action: Action, *, budget: Budget, task: dict) -> tuple[Receip
self.screenshot_counter += 1
return receipt, self._observation(task, step_idx, budget, receipt)
+ def observe(self, task, *, budget, previous):
+ return self._observation(task, previous.step_idx, budget, previous.last_receipt)
+
def export_outputs(self, paths: list[str], episode_dir) -> list[dict]:
records = []
for index, path in enumerate(paths):
diff --git a/src/tandem_eval/osworld.py b/src/tandem_eval/osworld.py
index f8d3905..064889d 100644
--- a/src/tandem_eval/osworld.py
+++ b/src/tandem_eval/osworld.py
@@ -16,6 +16,7 @@
import struct
import sys
import time
+import xml.etree.ElementTree as ET
from collections.abc import Callable
from pathlib import Path
from typing import Any
@@ -39,6 +40,20 @@
RECEIPT_TAIL = 12000
+_FOREGROUND_QUERY = """import json
+from Xlib.display import Display
+d=Display()
+p=d.screen().root.get_full_property(d.intern_atom('_NET_ACTIVE_WINDOW'),0)
+info={'foreground_title':'','foreground_class':[]}
+if p is not None and int(p.value[0]):
+ w=d.create_resource_object('window',int(p.value[0]))
+ title=w.get_full_property(d.intern_atom('_NET_WM_NAME'),0)
+ info={'foreground_title':title.value.decode('utf-8','replace') if title else str(w.get_wm_name() or ''),
+ 'foreground_class':list(w.get_wm_class() or [])}
+print(json.dumps(info))
+d.close()
+"""
+
def png_size(data: bytes) -> tuple[int, int]:
width, height = struct.unpack(">II", data[16:24])
@@ -47,8 +62,9 @@ def png_size(data: bytes) -> tuple[int, int]:
def gui_code(action: GuiAction) -> str:
params = action.parameters
- if action.action_type in {"click", "double_click"}:
- method = "click" if action.action_type == "click" else "doubleClick"
+ if action.action_type in {"click", "double_click", "right_click"}:
+ method = {"click": "click", "double_click": "doubleClick", "right_click": "rightClick"}[
+ action.action_type]
return f"pyautogui.{method}(x={params['x']}, y={params['y']})"
if action.action_type == "hotkey":
steps = []
@@ -66,7 +82,12 @@ def gui_code(action: GuiAction) -> str:
steps.append("pyautogui.press('f2')")
if params.get("input_mode", "replace") == "replace":
steps.append("pyautogui.hotkey('ctrl', 'a')")
- steps.append(f"pyautogui.write({params['text']!r})")
+ if params["text"].isascii():
+ steps.append(f"pyautogui.write({params['text']!r}, interval=0.03)")
+ else:
+ steps.append("__import__('subprocess').run(['xclip','-selection','clipboard','-t','UTF8_STRING'],"
+ f"input={params['text']!r}.encode('utf-8'),check=True)")
+ steps.append("pyautogui.hotkey('ctrl','v')")
return "; ".join(steps)
raise ValueError(f"不支持的 GUI 动作类型: {action.action_type}")
@@ -126,6 +147,7 @@ def _bootstrap_runtime(self) -> None:
os.environ["OSWORLD_FILE_BASE_URL"] = str(self.asset_root)
self.env: Any = None
self._step_seq = 0
+ self._frame_seq = 0
self.last_screenshot: bytes = b""
self.last_a11y: str = ""
self.last_browser_snapshot: dict | None = None
@@ -182,13 +204,15 @@ def _observation(self, task: dict, step_idx: int, budget: Budget,
receipt: Receipt | None) -> Observation:
stage = self.run_root / "obs_stage" / str(task["id"])
stage.mkdir(parents=True, exist_ok=True)
- shot_path = stage / f"step_{step_idx:02d}.png"
+ self._frame_seq += 1
+ frame_name = f"step_{step_idx:02d}_obs_{self._frame_seq:04d}"
+ shot_path = stage / f"{frame_name}.png"
shot_path.write_bytes(self.last_screenshot)
a11y_path: str | None = None
digest = node_count = None
present = bool(self.last_a11y.strip())
if present:
- a11y_path = str(stage / f"step_{step_idx:02d}.xml")
+ a11y_path = str(stage / f"{frame_name}.xml")
Path(a11y_path).write_text(self.last_a11y, encoding="utf-8")
digest = hashlib.sha256(self.last_a11y.encode()).hexdigest()
node_count = self.last_a11y.count("<")
@@ -196,7 +220,7 @@ def _observation(self, task: dict, step_idx: int, budget: Budget,
browser_snapshot = None
if self.last_browser_snapshot is not None:
serialized = json.dumps(self.last_browser_snapshot, ensure_ascii=False)
- browser_path = stage / f"step_{step_idx:02d}.dom.json"
+ browser_path = stage / f"{frame_name}.dom.json"
browser_path.write_text(serialized, encoding="utf-8")
browser_snapshot = BrowserSnapshot(
present=True,
@@ -220,15 +244,36 @@ def _observation(self, task: dict, step_idx: int, budget: Budget,
)
def _capture(self) -> None:
- obs = self.env._get_obs()
+ for attempt in range(3):
+ obs = self.env._get_obs()
+ if obs.get("screenshot"):
+ break
+ if attempt < 2:
+ time.sleep(1)
+ else:
+ raise RuntimeError("Environment returned no screenshot after three read-only observations")
self.last_screenshot = obs["screenshot"]
self.last_a11y = obs.get("accessibility_tree") or ""
+ result = self.env.controller.execute_python_command(_FOREGROUND_QUERY)
+ if result is None or result.get("returncode", 0):
+ raise RuntimeError("Read-only foreground-window observation failed")
+ info = json.loads(result.get("output") or "{}")
+ if self.last_a11y:
+ tree = ET.fromstring(self.last_a11y)
+ tree.set("foreground_title", info.get("foreground_title", ""))
+ tree.set("foreground_class", json.dumps(info.get("foreground_class", [])))
+ self.last_a11y = ET.tostring(tree, encoding="unicode")
self.last_browser_snapshot, diagnostic = capture_browser_snapshot(self.env.controller)
if not self._cdp_reported:
self._cdp_reported = True
public = {key: value for key, value in diagnostic.items() if key != "url"}
print(json.dumps({"event": "cdp_snapshot", **public}, ensure_ascii=False), flush=True)
+ def observe(self, task, *, budget, previous):
+ """Refresh screenshot, A11y and CDP without executing or replaying any action."""
+ self._capture()
+ return self._observation(task, previous.step_idx, budget, previous.last_receipt)
+
# ------------------------------------------------------------------ 协议
def _seed_setup_cache(self, task_id: str) -> None:
"""把共享盘已缓存的 setup 文件播种进本次 evaluation_cache(旧 runtime.py:532-539),
@@ -291,9 +336,17 @@ def acknowledged(command):
stdout = result.get("output", "")
stderr = result.get("error", "")
returncode = int(result.get("returncode", 0))
+ output_ref = None
+ if kind == "cli":
+ output = self.run_root / "cli_outputs" / f"action_{self._step_seq:04d}.json"
+ output.parent.mkdir(parents=True, exist_ok=True)
+ output.write_text(json.dumps({"stdout": stdout, "stderr": stderr,
+ "returncode": returncode}), encoding="utf-8")
+ output_ref = str(output)
return Receipt(
returncode=returncode, stdout=stdout[-RECEIPT_TAIL:], stderr=stderr[-RECEIPT_TAIL:],
kind=kind if kind in {"cli", "gui", "wait"} else "gui", acknowledged=True,
+ full_output_ref=output_ref,
)
def execute(self, action: Action, *, budget: Budget, task: dict) -> tuple[Receipt, Observation]:
diff --git a/src/tandem_eval/recorder.py b/src/tandem_eval/recorder.py
index fe26c6c..2df4145 100644
--- a/src/tandem_eval/recorder.py
+++ b/src/tandem_eval/recorder.py
@@ -57,6 +57,10 @@ def __init__(self, episode_dir: Path, task: dict, *, a11y_full: bool = False):
self.dir.mkdir(parents=True, exist_ok=True)
self.task = task
self.a11y_full = a11y_full
+ self.executed_steps = 0
+ self.environment_seconds = {}
+ self.model_usage = {}
+ self.fast_model = {"choice_cycles": 0, "seconds": 0.0, "cycles_with_image": 0}
self.started = time.perf_counter()
self._traj = (self.dir / "traj.jsonl").open("w", encoding="utf-8")
self._events = (self.dir / "events.jsonl").open("w", encoding="utf-8")
@@ -65,6 +69,7 @@ def __init__(self, episode_dir: Path, task: dict, *, a11y_full: bool = False):
# ------------------------------------------------------------------ 逐步记录
def traj(self, observation: Observation, action: Action, receipt: Receipt | None) -> None:
+ self.executed_steps += 1
browser_snapshot = observation.browser_snapshot
browser_record = browser_snapshot.to_dict() if browser_snapshot else None
if browser_snapshot and browser_snapshot.present and browser_snapshot.path:
@@ -89,9 +94,40 @@ def traj(self, observation: Observation, action: Action, receipt: Receipt | None
self._traj.flush()
def event(self, event: dict) -> None:
+ if event.get("route") == "environment_timing":
+ phase = event["phase"]
+ self.environment_seconds[phase] = self.environment_seconds.get(phase, 0.0) + event["seconds"]
+ if event.get("route") == "gui" and event.get("ranking"):
+ ranking = event["ranking"]
+ self.fast_model["choice_cycles"] += 1
+ self.fast_model["seconds"] += ranking["latency_ms"] / 1000
+ self.fast_model["cycles_with_image"] += int(ranking["used_image"])
+ if event.get("route") == "stage_verified":
+ event = self._persist_evidence(event)
self._events.write(json.dumps(event, ensure_ascii=False) + "\n")
self._events.flush()
+ def task_memory(self, state: dict) -> None:
+ """Persist the latest host memory and retain its transient evidence before cleanup."""
+ _write_json(self.dir / "task_memory.json", self._persist_evidence(state))
+
+ def _persist_evidence(self, state):
+ def persist(value):
+ if isinstance(value, dict):
+ return {k: persist(v) for k, v in value.items()}
+ if isinstance(value, list):
+ return [persist(v) for v in value]
+ if (isinstance(value, str) and any(part in value for part in ("/obs_stage/", "/cli_outputs/"))
+ and Path(value).is_file()):
+ target = self.dir / "evidence" / Path(value).name
+ target.parent.mkdir(exist_ok=True)
+ if not target.exists():
+ shutil.copy2(value, target)
+ return str(target)
+ return value
+
+ return persist(state)
+
def record_llm_call(self, *, role: str, request: Any, response: Any, **meta: Any) -> None:
"""慢模型客户端的 llm_sink;全量请求/响应落盘,密钥脱敏。"""
self._llm.write(
@@ -110,6 +146,16 @@ def record_llm_call(self, *, role: str, request: Any, response: Any, **meta: Any
+ "\n"
)
self._llm.flush()
+ stats = self.model_usage.setdefault(role, {"calls": 0, "seconds": 0.0, "prompt_tokens": 0,
+ "completion_tokens": 0, "reasoning_tokens": 0, "cached_tokens": 0, "calls_missing_usage": 0})
+ stats["calls"] += 1
+ stats["seconds"] += meta.get("elapsed_seconds", 0.0)
+ usage = response.get("usage") or {}
+ stats["calls_missing_usage"] += int(not usage)
+ for key in ("prompt_tokens", "completion_tokens"):
+ stats[key] += usage.get(key, 0)
+ stats["reasoning_tokens"] += (usage.get("completion_tokens_details") or {}).get("reasoning_tokens", 0)
+ stats["cached_tokens"] += (usage.get("prompt_tokens_details") or {}).get("cached_tokens", 0)
def screenshot(self, step_idx: int, data: bytes) -> None:
target = self.dir / "screenshots" / f"step_{step_idx:02d}.png"
@@ -150,7 +196,7 @@ def finalize(
"task_id": self.task["id"],
"harness": f"tandem {__version__}",
"stop_reason": stop_reason,
- "steps": steps,
+ "steps": self.executed_steps if exception is not None else steps,
"status": status,
"score": score,
"exception_type": type(exception).__name__ if exception else None,
@@ -163,6 +209,9 @@ def finalize(
self.dir / "timing.json",
{
"wall_seconds": round(time.perf_counter() - self.started, 3),
+ "environment_seconds": self.environment_seconds,
+ "model_usage_by_role": self.model_usage,
+ "fast_model": self.fast_model,
"status": status,
},
)
diff --git a/src/tandem_eval/rjob_spec.py b/src/tandem_eval/rjob_spec.py
index 99250ba..0e8a1fc 100644
--- a/src/tandem_eval/rjob_spec.py
+++ b/src/tandem_eval/rjob_spec.py
@@ -113,10 +113,17 @@ def _harness_config_section(config: dict, slow_api: dict, fast_api: dict) -> dic
return {
"execution_mode": execution.get("mode", "step"),
"stage_max_actions": execution.get("stage_max_actions", 24),
+ "goal_recovery_max_calls": int(execution.get("recovery_max_calls", 2)),
+ "goal_recovery_max_seconds": float(execution.get("recovery_max_seconds", 300)),
"route_policy": route.get("policy", "auto"),
"allow_cli": bool(route.get("allow_cli", True)),
"cli_timeout_seconds": int(route.get("cli_timeout_seconds", 90)),
"slow_max_tokens": int(slow_api.get("max_tokens", 16384)),
+ "slow_verify_max_tokens": int(execution.get("verify_max_tokens", 16384)),
+ "slow_verify_reasoning_effort": execution.get("verify_reasoning_effort"),
+ "stage_verify_max_chars": int(execution.get("verify_max_chars", 48000)),
+ "slow_image_quality": int(execution.get("image_quality", 85)),
+ "slow_max_request_bytes": int(execution.get("max_request_bytes", 2 * 1024 * 1024)),
"slow_timeout_seconds": float(slow_api.get("timeout_seconds", 120)),
"slow_reasoning_effort": slow_api.get("reasoning_effort"),
"slow_direct_api": bool(slow_api.get("direct_api", False)),
@@ -193,7 +200,8 @@ def build_job_spec(*, config: dict, run_id: str, repo_root: Path,
"image_stage": _image_stage_section(config),
"harness": {
"name": "tandem",
- "mode": "stage" if harness_config["execution_mode"] == "stage" else "planner_led",
+ "mode": (harness_config["execution_mode"]
+ if harness_config["execution_mode"] in {"stage", "goal"} else "planner_led"),
"models": {"slow": slow_name, "fast": fast_name},
},
"harness_config": harness_config,
diff --git a/src/tandem_eval/runner.py b/src/tandem_eval/runner.py
index 8a9d18c..872fefe 100644
--- a/src/tandem_eval/runner.py
+++ b/src/tandem_eval/runner.py
@@ -7,7 +7,9 @@
from dataclasses import replace
from pathlib import Path
+from tandem.models.slow import TaskBudgetExceeded
from tandem.protocol.models import Observation
+from tandem.runtime.goal_stage import ObservationRequired
from tandem.runtime.session import TandemSession
from .env import budget_of
@@ -20,12 +22,21 @@
def run_episode(env, session: TandemSession, recorder: EpisodeRecorder, task: dict,
*, max_steps: int, max_wall_seconds: float,
- extra_exports: list[str] | None = None) -> dict:
+ extra_exports: list[str] | None = None, started_monotonic: float | None = None) -> dict:
"""单题循环:obs → session.act → env.execute → 回执灌回,终态导出交付物并评分。"""
- started = time.perf_counter()
+ started = time.perf_counter() if started_monotonic is None else started_monotonic
steps_used = 0
stop_reason = "max_steps"
- observation: Observation = env.reset(task)
+
+ def environment_call(phase, function, *args, **kwargs):
+ before = time.perf_counter()
+ try:
+ return function(*args, **kwargs)
+ finally:
+ recorder.event({"route": "environment_timing", "phase": phase,
+ "seconds": time.perf_counter() - before})
+
+ observation: Observation = environment_call("setup", env.reset, task)
outputs: list[dict] = []
events_start = len(session.events) # 本题事件起点:act 后增量落盘(planner/快模型/验证)
@@ -34,6 +45,8 @@ def drain_events() -> None:
for event in session.events[events_start:]:
recorder.event(event)
events_start = len(session.events)
+ if session.goal is not None:
+ recorder.task_memory(session.goal.memory.snapshot())
while True:
budget = budget_of(
@@ -50,19 +63,32 @@ def drain_events() -> None:
stop_reason = "wall_time_limit"
break
observation = replace(observation, budget=budget)
- action = session.act(observation)
- drain_events()
+ try:
+ action = session.act(observation)
+ except ObservationRequired as refresh:
+ recorder.event({"route": "observation_refresh", "step": steps_used,
+ "reason": str(refresh), "action_replayed": False})
+ observation = environment_call("observe", env.observe, task, budget=budget, previous=observation)
+ continue
+ except TaskBudgetExceeded as exhausted:
+ stop_reason = "wall_time_limit"
+ recorder.event({"route": "task_budget_exhausted", "step": steps_used,
+ "reason": str(exhausted), "next": "official_evaluation"})
+ break
+ finally:
+ drain_events()
recorder.event({"step": steps_used, "kind": "action", "action": action.to_dict()})
if getattr(action, "kind", "") in TERMINAL_KINDS:
stop_reason = "done" if action.kind == "finish" else action.reason
# 终态仍须送达 VM(DONE/FAIL 触发评分钩子,旧 runtime 同款)
- env.execute(action, budget=budget, task=task)
+ environment_call("execute", env.execute, action, budget=budget, task=task)
if action.kind == "finish":
- outputs = env.export_outputs(
+ outputs = environment_call(
+ "export", env.export_outputs,
list(action.outputs) + list(extra_exports or []), recorder.dir
)
break
- receipt, observation = env.execute(action, budget=budget, task=task)
+ receipt, observation = environment_call("execute", env.execute, action, budget=budget, task=task)
recorder.traj(observation, action, receipt)
# 环境提供素材属性时写入 episode 目录(真实环境 M3 同样暴露 last_* 属性)
shot = getattr(env, "last_screenshot", None)
@@ -72,7 +98,9 @@ def drain_events() -> None:
if a11y_text:
recorder.a11y(steps_used, a11y_text)
steps_used += 1
- raw = env.score(task)
+ raw = environment_call("evaluate", env.score, task)
+ if session.goal is not None:
+ recorder.task_memory(session.goal.memory.snapshot())
score = official_score(raw) if raw is not None else None
return recorder.finalize(
stop_reason=stop_reason,
@@ -101,6 +129,7 @@ def run(spec: dict, *, env, session: TandemSession, tasks: list[TaskRef],
recorder: EpisodeRecorder | None = None
error: BaseException | None = None
task: dict = {}
+ started = time.perf_counter()
try:
task = env.fetch_task(ref.task_id, ref.domain)
recorder = EpisodeRecorder(
@@ -108,11 +137,13 @@ def run(spec: dict, *, env, session: TandemSession, tasks: list[TaskRef],
)
if llm_tap is not None:
llm_tap.current = recorder.record_llm_call
+ if session.components.set_deadline is not None:
+ session.components.set_deadline(started + max_wall)
session.begin(task["instruction"], spec.get("requirements"))
episodes.append(
run_episode(env, session, recorder, task,
max_steps=max_steps, max_wall_seconds=max_wall,
- extra_exports=list(spec.get("export_paths", [])))
+ extra_exports=list(spec.get("export_paths", [])), started_monotonic=started)
)
except BaseException as exc: # 单题失败不终止整个 run
error = exc
@@ -129,8 +160,13 @@ def run(spec: dict, *, env, session: TandemSession, tasks: list[TaskRef],
)
)
finally:
+ if recorder is not None and session.goal is not None:
+ recorder.task_memory(session.goal.memory.snapshot())
if llm_tap is not None:
llm_tap.current = None
+ if session.components.set_deadline is not None:
+ session.components.set_deadline(None)
+ write_summary(run_root, episodes, spec)
# obs_stage 是 episode 内工作区(截图正本已由 recorder 写入 episode 目录,
# a11y 摘要已入 traj);episode 结束即清理,避免双份存储
staging = run_root / "obs_stage" / str(ref.task_id)
diff --git a/tests/eval/test_browser.py b/tests/eval/test_browser.py
index 2bb9219..2a4961e 100644
--- a/tests/eval/test_browser.py
+++ b/tests/eval/test_browser.py
@@ -33,6 +33,49 @@ def test_cdp_failure_returns_a11y_fallback_diagnostic():
assert diagnostic["available"] is False and "connection refused" in diagnostic["error"]
+def test_readonly_javascript_collects_clickable_cards_but_not_their_children_or_occluded_controls():
+ import shutil
+ import subprocess
+
+ import pytest
+
+ node = shutil.which("node")
+ if node is None:
+ pytest.skip("Optional Node runtime is unavailable")
+ namespace = {"__name__": "cdp_test"}
+ exec(VM_SCRIPT, namespace)
+ setup = r'''
+ const body = {tabIndex:-1, onclick:null, cursor:'auto', closest:()=>null,
+ matches:s=>s.includes('body')};
+ const make = (text, x, parent=body, native=false) => ({
+ tagName:native?'BUTTON':'DIV', type:'', tabIndex:-1, onclick:native?null:()=>{},
+ cursor:native?'auto':'pointer', parentElement:parent, childNodes:[{nodeType:3,textContent:text}],
+ labels:[], innerText:text, disabled:false, checkVisibility:()=>true,
+ getAttribute:()=>null, matches:s=>native&&s.includes('button'), closest:()=>null,
+ querySelector:()=>null, querySelectorAll:()=>[],
+ getBoundingClientRect:()=>({left:x,top:20,right:x+100,bottom:60,width:100,height:40}),
+ contains(other){return other===this||other.parentElement===this;},
+ cloneNode(){return {innerText:text,querySelectorAll:()=>[]};}
+ });
+ const card=make('Afternoon class',20);
+ const child=make('Child label',20,card);
+ const hidden=make('Covered button',200,body,true);
+ const cover=make('Overlay',200);
+ global.getComputedStyle=e=>({cursor:e.cursor,display:'block',visibility:'visible',opacity:'1'});
+ global.innerWidth=800;global.innerHeight=600;
+ global.window={screenX:0,screenY:0,outerWidth:800,innerWidth:800,outerHeight:600,innerHeight:600};
+ global.location={href:'https://example.test/'};global.devicePixelRatio=1;
+ global.document={title:'Synthetic cards',visibilityState:'visible',activeElement:null,
+ getElementById:()=>null,
+ querySelectorAll:s=>s.startsWith('a[href]')?[card,child,hidden]:[],
+ elementFromPoint:x=>x<150?child:cover};
+ '''
+ run = subprocess.run([node, "-e", setup + "\nconsole.log(JSON.stringify(" + namespace["EXPRESSION"]
+ + "));"], capture_output=True, text=True, check=False)
+ assert run.returncode == 0, run.stderr
+ assert [e["name"] for e in json.loads(run.stdout)["elements"]] == ["Afternoon class"]
+
+
def test_vm_websocket_frame_parser_executes_with_stdlib_only():
namespace = {"__name__": "cdp_test"}
exec(VM_SCRIPT, namespace)
diff --git a/tests/eval/test_config.py b/tests/eval/test_config.py
index 5739758..7b5f0fd 100644
--- a/tests/eval/test_config.py
+++ b/tests/eval/test_config.py
@@ -69,8 +69,9 @@ def test_models_are_named_api_registries():
for api in slow["apis"].values():
assert set(api) >= {"base_url", "api_key", "model"}
for api in fast["apis"].values():
- assert api["backend"] in {"jev", "kev"}
- assert api.get("endpoint") or api.get("endpoint_file")
+ assert api["backend"] in {"jev", "kev", "planner"}
+ if api["backend"] != "planner":
+ assert api.get("endpoint") or api.get("endpoint_file")
def test_paths_are_absolute_and_credentials_inlined():
diff --git a/tests/eval/test_experiment_script.py b/tests/eval/test_experiment_script.py
index 9c2d3ea..9773b32 100644
--- a/tests/eval/test_experiment_script.py
+++ b/tests/eval/test_experiment_script.py
@@ -54,6 +54,16 @@ def test_invalid_experiment_name_is_rejected(tmp_path):
assert not (tmp_path / "bad name").exists()
+def test_dry_run_selects_goal_mode_without_changing_script_default(tmp_path):
+ result = subprocess.run([
+ "bash", str(SCRIPT), "--experiment", "goal-smoke", "--tasks", "008", "--execution-mode", "goal",
+ "--stage-max-actions", "2", "--dry-run"], capture_output=True, text=True,
+ env={**os.environ, "TANDEM_EXPERIMENT_ROOT": str(tmp_path)})
+ assert result.returncode == 0, result.stderr
+ manifest = json.loads((tmp_path / "goal-smoke/manifest.json").read_text())
+ assert manifest["execution_mode"] == "goal" and manifest["stage_max_actions"] == 2
+
+
def test_dry_run_uses_example_when_private_config_is_missing(tmp_path):
env = {
**os.environ,
diff --git a/tests/eval/test_mock_chain.py b/tests/eval/test_mock_chain.py
index 5391883..a0a4933 100644
--- a/tests/eval/test_mock_chain.py
+++ b/tests/eval/test_mock_chain.py
@@ -41,9 +41,10 @@ def _spec(tmp_path: Path, **overrides) -> dict:
return base
-@pytest.mark.parametrize("mode", ["step", "stage"])
+@pytest.mark.parametrize("mode", ["step", "stage", "goal"])
def test_full_mock_chain_produces_renderable_result_dir(tmp_path, mode):
- episodes = run_from_spec(_spec(tmp_path, harness_config={"execution_mode": mode}))
+ episodes = run_from_spec(_spec(tmp_path, harness_config={"execution_mode": mode},
+ requirements=None if mode == "goal" else []))
assert episodes and episodes[0]["status"] == "scored"
run_root = tmp_path / "run"
episode_dir = run_root / "episodes" / "mock-mock-000"
@@ -64,6 +65,12 @@ def test_full_mock_chain_produces_renderable_result_dir(tmp_path, mode):
assert episode["stop_reason"] == "done" and episode["score"] == 1.0
summary = json.loads((run_root / "summary.json").read_text())
assert summary["scored"] == 1 and summary["average_score"] == 1.0
+ if mode == "goal":
+ memory = json.loads((episode_dir / "task_memory.json").read_text())
+ assert memory["work_items"][0]["status"] == "done"
+ events = [json.loads(line) for line in (episode_dir / "events.jsonl").read_text().splitlines()]
+ assert any(e.get("route") == "stage_verified" for e in events)
+ assert any(e.get("route") == "observation_refresh" for e in events)
# 渲染(demo 占位)可离线打开
page = render_summary.render(run_root)
@@ -95,6 +102,51 @@ def fetch_task(self, task_id: str, domain: str = "") -> dict:
assert summary["failed_unscored"] == 1 and summary["average_score"] == 1.0
+def test_agent_budget_exhausted_inside_model_still_reaches_official_scoring(tmp_path):
+ from tandem.config import SessionConfig
+ from tandem.models.slow import TaskBudgetExceeded
+ from tandem.runtime.session import TandemSession
+
+ def planner(*args, **kwargs):
+ raise TaskBudgetExceeded("Agent task budget exhausted")
+
+ components = make_components(planner, lambda *a, **kw: None, FixedSelector())
+ root = tmp_path / "run"
+ episodes = run(_spec(tmp_path), env=MockEnv(base_dir=root / "mock_env"),
+ session=TandemSession(components, config=SessionConfig()),
+ tasks=[TaskRef("one", "mock")], run_root=root)
+ assert episodes[0]["stop_reason"] == "wall_time_limit"
+ assert episodes[0]["status"] == "scored"
+ assert episodes[0]["score"] is not None
+
+
+def test_requirement_timeout_has_a_deadline_and_flushes_summary_before_next_task(tmp_path):
+ from tandem.config import SessionConfig
+ from tandem.runtime.session import TandemSession
+
+ deadlines = []
+ root = tmp_path / "run"
+
+ class Env(MockEnv):
+ def fetch_task(self, task_id, domain=""):
+ if task_id == "second":
+ assert json.loads((root / "summary.json").read_text())["failed_unscored"] == 1
+ return super().fetch_task(task_id, domain)
+
+ def requirements(instruction):
+ assert isinstance(deadlines[-1], float)
+ raise TimeoutError("Task deadline exhausted during requirements")
+
+ components = make_components(lambda *a, **kw: None, lambda *a, **kw: None, FixedSelector())
+ components.requirements = requirements
+ components.set_deadline = deadlines.append
+ episodes = run(_spec(tmp_path, requirements=None), env=Env(base_dir=root / "mock_env"),
+ session=TandemSession(components, config=SessionConfig()),
+ tasks=[TaskRef("first"), TaskRef("second")], run_root=root)
+ assert [item["status"] for item in episodes] == ["failed_unscored", "failed_unscored"]
+ assert deadlines[1] is None and deadlines[3] is None
+
+
def test_entry_cli_from_spec_file(tmp_path, capsys):
spec_path = tmp_path / "spec.json"
spec_path.write_text(json.dumps(_spec(tmp_path), ensure_ascii=False), encoding="utf-8")
diff --git a/tests/eval/test_osworld_env.py b/tests/eval/test_osworld_env.py
index 10fe121..ed7bdec 100644
--- a/tests/eval/test_osworld_env.py
+++ b/tests/eval/test_osworld_env.py
@@ -4,6 +4,7 @@
import json
import struct
+import xml.etree.ElementTree as ET
from pathlib import Path
from types import ModuleType
@@ -42,10 +43,10 @@ def test_gui_code_matches_legacy_pyautogui_semantics():
assert gui_code(GuiAction("type", {
"text": "42", "input_mode": "replace", "click": [650, 102], "press_f2": True,
})) == ("pyautogui.click(x=650, y=102); pyautogui.press('f2'); "
- "pyautogui.hotkey('ctrl', 'a'); pyautogui.write('42')")
+ "pyautogui.hotkey('ctrl', 'a'); pyautogui.write('42', interval=0.03)")
assert gui_code(GuiAction("type", {
"text": "cmd", "input_mode": "insert", "click": None, "press_f2": False,
- })) == "pyautogui.write('cmd')"
+ })) == "pyautogui.write('cmd', interval=0.03)"
def test_action_code_cli_wait_and_terminals():
@@ -62,6 +63,64 @@ def test_png_size_parses_ihdr():
assert png_size(_png(1920, 1080)) == (1920, 1080)
+def test_read_only_refresh_keeps_cdp_receipt_and_prior_evidence(tmp_path, env, monkeypatch):
+ snapshot = {"elements": [{"index": 1, "role": "entry", "name": "Receipt Date"}]}
+ monkeypatch.setattr("tandem_eval.osworld.capture_browser_snapshot",
+ lambda controller: (snapshot, {"status": "ok"}))
+ task = _task_file(tmp_path)
+ first = env.reset(task)
+ budget = Budget(steps_remaining=10, wall_seconds_remaining=60)
+ receipt, previous = env.execute(GuiAction("click", {"x": 1, "y": 2}), budget=budget, task=task)
+ steps, commands = list(env.env.steps), list(env.env.controller.commands)
+ before = Path(previous.screenshot.path).read_bytes()
+ refreshed = env.observe(task, budget=budget, previous=previous)
+ assert refreshed.protocol_version == "0.2" and refreshed.last_receipt is receipt
+ assert refreshed.step_idx == previous.step_idx
+ assert env.env.steps == steps
+ assert env.env.controller.commands[:-1] == commands
+ assert "_NET_ACTIVE_WINDOW" in env.env.controller.commands[-1] # Read-only window observation.
+ tree = ET.fromstring(Path(refreshed.accessibility_tree.path).read_text())
+ assert tree.get("foreground_title") == "main-window"
+ assert json.loads(tree.get("foreground_class")) == ["app", "App"]
+ assert refreshed.browser_snapshot.source == "cdp"
+ assert json.loads(Path(refreshed.browser_snapshot.path).read_text()) == snapshot
+ assert refreshed.screenshot.path != previous.screenshot.path != first.screenshot.path
+ assert Path(previous.screenshot.path).read_bytes() == before
+
+
+def test_empty_screenshot_retries_only_observation(tmp_path, env, monkeypatch):
+ from unittest.mock import Mock
+
+ task = _task_file(tmp_path)
+ previous = env.reset(task)
+ valid = env.env._get_obs()
+ observe = Mock(side_effect=[{"screenshot": None}, {"screenshot": b""}, valid])
+ monkeypatch.setattr(env.env, "_get_obs", observe)
+ monkeypatch.setattr("tandem_eval.osworld.time.sleep", lambda seconds: None)
+ budget = Budget(steps_remaining=5, wall_seconds_remaining=30)
+ refreshed = env.observe(task, budget=budget, previous=previous)
+ assert observe.call_count == 3 and env.env.steps == []
+ assert refreshed.step_idx == previous.step_idx and refreshed.last_receipt is None
+ assert Path(refreshed.screenshot.path).read_bytes() == valid["screenshot"]
+
+
+def test_repeated_empty_screenshot_is_explicit_observation_failure(tmp_path, env, monkeypatch):
+ from unittest.mock import Mock
+
+ task = _task_file(tmp_path)
+ previous = env.reset(task)
+ observe = Mock(return_value={"screenshot": None})
+ monkeypatch.setattr(env.env, "_get_obs", observe)
+ monkeypatch.setattr("tandem_eval.osworld.time.sleep", lambda seconds: None)
+ with pytest.raises(RuntimeError, match="no screenshot after three read-only"):
+ env.observe(task, budget=previous.budget, previous=previous)
+ assert observe.call_count == 3 and env.env.steps == []
+
+
+def test_right_click_codegen_uses_real_mouse_button():
+ assert gui_code(GuiAction("right_click", {"x": 20, "y": 30})) == "pyautogui.rightClick(x=20, y=30)"
+
+
# ------------------------------------------------------------------ adapter
class FakeController:
def __init__(self):
@@ -71,6 +130,9 @@ def __init__(self):
def execute_python_command(self, command):
self.commands.append(command)
+ if "_NET_ACTIVE_WINDOW" in command:
+ return {"returncode": 0, "output": json.dumps({
+ "foreground_title": "main-window", "foreground_class": ["app", "App"]})}
return {"returncode": 0, "output": json.dumps(
{"returncode": 0, "stdout": "ok-out", "stderr": ""})}
diff --git a/tests/eval/test_rjob_client.py b/tests/eval/test_rjob_client.py
index 53dd21d..734284f 100644
--- a/tests/eval/test_rjob_client.py
+++ b/tests/eval/test_rjob_client.py
@@ -146,6 +146,18 @@ def test_cli_budget_override_merges_with_config(tmp_path):
assert spec["budget"] == {"max_steps": 3, "max_wall_seconds": 60.0}
+def test_goal_recovery_budget_is_taken_from_execution_config(tmp_path):
+ config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy"))
+ config["harness"]["execution"] = {
+ "mode": "goal", "recovery_max_calls": 3, "recovery_max_seconds": 240,
+ }
+ spec = build_job_spec(config=config, run_id="goal", repo_root=tmp_path,
+ spec_overrides={"benchmark": "mock"})
+ assert spec["harness_config"]["execution_mode"] == "goal"
+ assert spec["harness_config"]["goal_recovery_max_calls"] == 3
+ assert spec["harness_config"]["goal_recovery_max_seconds"] == 240
+
+
def test_watch_until_terminal_and_timeout():
runner = FakeRunner({"get": ["running", "succeeded"]})
client = RjobClient(runner)
diff --git a/tests/harness/conftest.py b/tests/harness/conftest.py
index f7317ab..e91bc14 100644
--- a/tests/harness/conftest.py
+++ b/tests/harness/conftest.py
@@ -7,6 +7,7 @@
from pathlib import Path
import pytest
+from PIL import Image
from tandem.components import Components
from tandem.gui.candidates import ActionCandidateBuilder
@@ -61,7 +62,7 @@ def make_observation(tmp_path: Path, step_idx: int, *, receipt: Receipt | None =
tree_xml: str | None = A11Y_XML, steps: int = 50,
wall: float = 1800.0, screenshot_sha: str = SHA) -> Observation:
shot = tmp_path / f"step_{step_idx:02d}.png"
- shot.write_bytes(b"png-bytes")
+ Image.new("RGB", (64, 64), "white").save(shot)
a11y_path = None
if tree_xml is not None:
a11y_path = str(tmp_path / f"a11y_{step_idx:02d}.xml")
diff --git a/tests/harness/test_execution_control.py b/tests/harness/test_execution_control.py
new file mode 100644
index 0000000..b75af0b
--- /dev/null
+++ b/tests/harness/test_execution_control.py
@@ -0,0 +1,184 @@
+"""Synthetic, task-independent execution-control regressions; not benchmark results."""
+
+from dataclasses import replace
+
+from tandem.gui.candidates import ActionCandidate
+from tandem.gui.compile import gui_action
+from tandem.gui.goal_candidates import goal_candidates
+from tandem.gui.interaction import GuiInteractionController
+from tandem.memory.context import execution_context, planning_context
+from tandem.memory.state import TaskMemory
+from tandem.planning.goal_planner import goal_error
+
+from .conftest import build_space
+from .test_goal_execution import _space
+from .test_goal_progress import FORM
+from .test_goal_stage import plan, requirements, result
+
+
+def _setup(inputs=()):
+ memory = TaskMemory("Edit and save", requirements())
+ memory.start({**plan(), "inputs": list(inputs)}, 0)
+ targets, _ = _space(FORM)
+ controller = GuiInteractionController()
+ space = build_space(targets)
+ controller.finalize(space, targets, memory.active_stage, memory, FORM)
+ return controller, memory, targets
+
+
+def _pending(controller, operation="keypress", sequence=None):
+ return {"operation": operation, "action": {"parameters": {"sequence": sequence}},
+ "control_scope": controller.control["scope_key"], "control_state": controller.content_state}
+
+
+def test_pixels_and_stage_rewording_do_not_release_a_failed_execution_path():
+ controller, memory, targets = _setup()
+ for _ in range(4):
+ controller.receive(_pending(controller, sequence=[["tab"]]), "unknown", True, 0, targets=targets)
+ assert controller.checkpoint_due
+ assert controller.local_effect == "unknown"
+ update = result(unit_met=False, delivery_state="none", stage_outcome="not_met")
+ controller.finish(memory, update, [{"evidence_ref": "A2"}])
+ memory.apply(update, {"A2": {}})
+ assert memory.work_items["R1:unit:1"]["recovery_required"]
+ assert "execution recovery" in goal_error(plan(), memory, True)
+ revised = {**plan(), "method": "Different words", "recovery_strategy": "alternate_interaction"}
+ assert not goal_error(revised, memory, True)
+ memory.start(revised, 4)
+ space = build_space(targets)
+ space.target_heads["keypress"] += [ActionCandidate(
+ "tab", "keypress", "Tab", metadata={"key_sequence": [["tab"]]})]
+ controller.finalize(space, targets, memory.active_stage, memory, FORM)
+ assert not any(c.id in {"tab", "KEYPRESS_TAB"} for c in space.candidates())
+ assert any(c.operation == "click" for c in space.candidates())
+ assert planning_context(memory)["execution_constraints"][0]["blocked_families"] == ["focus_traversal"]
+
+
+def test_new_known_focus_positions_are_allowed_until_the_traversal_cycles():
+ controller, memory, targets = _setup()
+ base = next(t for t in targets if t.name == "Title *")
+ fields = [replace(base, name=f"Field {i}", semantic_key=f"field-{i}", focused=False)
+ for i in range(8)]
+ controller.finalize(build_space(fields), fields, memory.active_stage, memory, FORM)
+ for index in range(8):
+ observed = [replace(t, focused=i == index) for i, t in enumerate(fields)]
+ controller.receive(_pending(controller, sequence=[["tab"]]), "unknown", True, 0, targets=observed)
+ assert not controller.checkpoint_due
+ observed = [replace(t, focused=i == 0) for i, t in enumerate(fields)]
+ controller.receive(_pending(controller, sequence=[["tab"]]), "unknown", True, 0, targets=observed)
+ assert controller.checkpoint_due
+
+
+def test_new_observed_content_allows_continuous_navigation_without_semantic_verify():
+ controller, memory, targets = _setup()
+ base = next(t for t in targets if t.name == "Title *")
+ for index in range(12):
+ observed = [replace(base, name=f"Visible content section {index}")]
+ controller.receive(_pending(controller, operation="scroll"), "unknown", True, 0, targets=observed)
+ controller.finalize(build_space(observed), observed, memory.active_stage, memory, FORM)
+ assert controller.local_effect == "controls_changed"
+ assert not controller.checkpoint_due and not controller.control["blocked_families"]
+
+
+def test_confirmed_field_effect_is_progress_but_does_not_erase_another_failed_path():
+ controller, _, targets = _setup()
+ controller.control.update(blocked_families=["scroll"], recovery_required=True)
+ pending = _pending(controller, operation="type_text")
+ controller.receive(pending, "unknown", True, 0, targets=targets)
+ assert controller.control["blocked_families"] == ["scroll"]
+ controller.receive(pending, "met", True, 0, targets=targets)
+ assert controller.local_effect == "confirmed_input"
+ assert controller.control["without_effect"] == 0
+ assert controller.control["blocked_families"] == ["scroll"]
+ assert controller.control["recovery_required"]
+
+
+def test_final_scope_rejects_a_commit_added_for_a_background_field():
+ value = {"label": "Command", "target_hint": "Title", "text": "Example(1)",
+ "input_mode": "replace", "kind": "command", "commit": "enter"}
+ controller, memory, targets = _setup([value])
+ # Establish a real command-input dispatch before the new foreground modal appears.
+ space = build_space(targets)
+ space.target_heads["type_text"] = [ActionCandidate(
+ "command", "type_text", "Type command", target_id=next(t.id for t in targets if t.name == "Title *"),
+ metadata={"text": value["text"], "target_hint": "Title", "input_mode": "replace"})]
+ controller.constrain(space, targets, memory.active_stage, memory)
+ typed = space.target_heads["type_text"][0]
+ controller.receive({"operation": "type_text",
+ "interaction_binding": typed.metadata["interaction_binding"]},
+ "unknown", True, 0)
+ modal = (''
+ '')
+ memory.active_stage.update(target_document="expected-document.pptx", document_recovery=True)
+ space = goal_candidates(build_space(targets), targets, modal, [])
+ space.target_heads["visual"] = [ActionCandidate(
+ "close", "click", "Close the visible confirmation", source="visual_grounder",
+ metadata={"visual_point": [650, 420], "blocking_overlay": True,
+ "overlay_name": "Confirmation", "overlay_bounds": [400, 400, 300, 200]})]
+ context = controller.finalize(space, targets, memory.active_stage, memory, modal)
+ assert not any(c.metadata.get("commit_action") for c in space.candidates())
+ assert "Confirmation" in context["next_intent"]
+ assert any(c.id == "close" for c in space.candidates())
+ assert gui_action(next(c for c in space.candidates() if c.id == "close"), None).action_type == "click"
+
+
+def test_non_delivery_source_facts_do_not_clear_an_active_modal_failure():
+ controller, memory, targets = _setup()
+ controller.control.update(global_scope=True, attempts={"focus_traversal": 5})
+ update = result(stage_outcome="met", unit_met=False, delivery_state="none", progress_kind="source",
+ facts=[{"key": "source.note", "value": "An unrelated observed note",
+ "status": "confirmed",
+ "scope": "task", "source_ref": "source.txt", "evidence_refs": ["A2"]}])
+ controller.finish(memory, update, [{"evidence_ref": "A2"}])
+ memory.apply(update, {"A2": {}})
+ assert controller.control["blocked_families"] == ["focus_traversal"]
+ assert controller.control["recovery_required"]
+
+
+def test_a_physical_modal_failure_survives_switching_work_units():
+ req = requirements()[0]
+ req.update(units=["First section", "Second section"], unit_fact_keys=[[], []])
+ memory = TaskMemory("Edit two sections", [req])
+ controller = GuiInteractionController()
+ modal = ''
+
+ def space():
+ value = goal_candidates(build_space([]), [], modal, [])
+ value.target_heads["visual"] = [ActionCandidate(
+ "visible-button", "click", "Visible button", source="visual_grounder",
+ metadata={"visual_point": [30, 40], "blocking_overlay": True,
+ "overlay_name": "Confirmation", "overlay_bounds": [10, 20, 100, 100]})]
+ return value
+
+ memory.start(plan(), 0)
+ controller.finalize(space(), [], memory.active_stage, memory, modal)
+ for _ in range(4):
+ controller.receive(_pending(controller, sequence=[["tab"]]), "unknown", True, 0)
+ update = result(stage_outcome="not_met", unit_met=False, delivery_state="none")
+ controller.finish(memory, update, [{"evidence_ref": "A2"}])
+ memory.apply(update, {"A2": {}})
+ memory.start({**plan(), "work_item_id": "R1:unit:2"}, 4)
+ choices = space()
+ controller.finalize(choices, [], memory.active_stage, memory, modal)
+ assert controller.control["global_scope"]
+ assert not any(c.id == "KEYPRESS_TAB" for c in choices.candidates())
+ assert execution_context(memory)["execution_constraints"][0]["blocked_families"] == ["focus_traversal"]
+
+
+def test_confirmed_required_source_content_can_release_nonmodal_browsing():
+ req = requirements()[0]
+ req.update(facts_needed=[{"key": "source.required", "description": "Required source section"}],
+ unit_fact_keys=[["source.required"]])
+ memory = TaskMemory("Read required source", [req])
+ memory.start({**plan(), "purpose": "prepare"}, 0)
+ controller = GuiInteractionController()
+ targets, _ = _space(FORM)
+ controller.finalize(build_space(targets), targets, memory.active_stage, memory, FORM)
+ controller.control.update(blocked_families=["scroll"], attempts={"scroll": 4}, recovery_required=True)
+ update = result(unit_met=False, delivery_state="none", progress_kind="source", facts=[{
+ "key": "source.required", "value": "Confirmed requested section", "status": "confirmed",
+ "scope": "task", "source_ref": "source.txt", "evidence_refs": ["A2"]}])
+ controller.finish(memory, update, [{"evidence_ref": "A2"}])
+ assert not controller.control["blocked_families"]
+ assert not controller.control["recovery_required"]
diff --git a/tests/harness/test_focus.py b/tests/harness/test_focus.py
index aa5744c..b312dfe 100644
--- a/tests/harness/test_focus.py
+++ b/tests/harness/test_focus.py
@@ -14,6 +14,14 @@ def test_menu_extraction_with_alt_mnemonic():
assert menus[0]["key_sequence"] == [["esc"], ["alt", "f"], ["i"]]
+def test_menu_item_mnemonic_does_not_confirm_the_dialog_it_opens():
+ tree = ''''''
+ _, menus = input_and_menu_context(tree)
+ assert menus[0]["key_sequence"] == [["esc"], ["alt", "f"], ["o"]]
+
+
def test_inputs_include_focused_editable_with_bounds():
inputs, _ = input_and_menu_context(A11Y_XML)
focused = [item for item in inputs if item["focused"]]
diff --git a/tests/harness/test_goal_blocker.py b/tests/harness/test_goal_blocker.py
new file mode 100644
index 0000000..ede21d0
--- /dev/null
+++ b/tests/harness/test_goal_blocker.py
@@ -0,0 +1,117 @@
+"""Generic binding/blocker regressions derived from recorded failures; synthetic fixtures."""
+
+from dataclasses import replace
+
+from tandem.gui.candidates import ActionCandidate
+from tandem.gui.goal_candidates import _input_target, goal_candidates
+from tandem.gui.interaction import GuiInteractionController
+from tandem.memory.context import execution_context, planning_context
+from tandem.memory.state import TaskMemory
+
+from .conftest import build_space
+from .test_goal_execution import _space
+from .test_goal_progress import FORM
+from .test_goal_stage import plan, requirements, result
+
+
+def test_whole_form_button_cannot_steal_native_choice_fields():
+ targets, _ = _space(FORM)
+ base = targets[0]
+ fields = [replace(base, id="country", role="combo_box", name="Country of Birth",
+ input_type="select-one", operations=("click",)),
+ replace(base, id="level", role="combo_box", name="Education Level",
+ input_type="select-one", operations=("click",)),
+ replace(base, id="form", role="push_button", operations=("click",), input_type="",
+ name="Questionnaire Country of Birth China Education Level Bachelor's Save Submit")]
+ for label, text, expected in [("Country of Birth", "China", "country"),
+ ("Education Level", "Bachelor's", "level")]:
+ value = {"label": label, "target_hint": label, "text": text,
+ "kind": "choice", "input_mode": "replace"}
+ assert _input_target(value, fields).id == expected
+
+
+def setup_controller():
+ req = requirements()[0]
+ req.update(units=["First unit", "Second unit"], unit_fact_keys=[[], []])
+ memory = TaskMemory("Complete two units", [req])
+ memory.start(plan(), 0)
+ controller = GuiInteractionController()
+ targets, _ = _space(FORM)
+ return controller, memory, targets
+
+
+def test_rejected_zero_action_completion_survives_switching_work_units():
+ controller, memory, targets = setup_controller()
+ space = goal_candidates(build_space(targets), targets, FORM, [])
+ controller.finalize(space, targets, memory.active_stage, memory, FORM, "S1:O0")
+ assert "done" in space.controls
+ rejected = result(stage_outcome="not_met", unit_met=False, delivery_state="none",
+ summary="The window was not maximized", evidence_refs=["S1:O0"])
+ controller.finish(memory, rejected, [])
+ memory.apply(rejected, {"S1:O0": {}})
+ memory.start({**plan(), "work_item_id": "R1:unit:2"}, 0)
+ space = goal_candidates(build_space(targets), targets, FORM, [])
+ controller.finalize(space, targets, memory.active_stage, memory, FORM, "S2:O0")
+ assert "done" not in space.controls
+ assert memory.active_blocker["kind"] == "rejected_completion"
+ assert execution_context(memory)["active_blocker"] == planning_context(memory)["active_blocker"]
+ assert controller.control["failed_stages"] == 1
+
+
+def test_new_related_control_value_releases_rejected_completion_with_evidence():
+ controller, memory, targets = setup_controller()
+ controller.finalize(goal_candidates(build_space(targets), targets, FORM, []),
+ targets, memory.active_stage, memory, FORM, "S1:O0")
+ controller.finish(memory, result(stage_outcome="not_met", unit_met=False, delivery_state="none"), [])
+ observed = [replace(t, value="New observed field value") for t in targets]
+ space = goal_candidates(build_space(observed), observed, FORM, [])
+ controller.finalize(space, observed, memory.active_stage, memory, FORM, "S1:O1")
+ assert "done" in space.controls
+ assert memory.active_blocker["status"] == "resolved"
+ assert memory.active_blocker["resolution_evidence_refs"] == ["S1:O1"]
+
+
+def test_short_goal_is_not_replaced_by_acceptance_criterion():
+ controller, memory, targets = setup_controller()
+ memory.active_stage.update(goal="Maximize the current window", completion_criteria="Window is maximized")
+ context = controller.finalize(goal_candidates(build_space(targets), targets, FORM, []),
+ targets, memory.active_stage, memory, FORM)
+ view = controller.choice_context({**execution_context(memory), **context})
+ assert view["objective"] == "Maximize the current window"
+ assert view["exit_condition"] == "Window is maximized"
+
+
+def test_unrelated_modal_names_share_one_owner_and_suspend_future_goal():
+ controller, memory, _ = setup_controller()
+ tree = ''
+ controls = []
+ for name in ["Default office software prompt", "Default software notice", "Editor prompt"]:
+ space = goal_candidates(build_space([]), [], tree, [])
+ space.target_heads["visual"] = [ActionCandidate("close", "click", "Close", source="visual_grounder",
+ metadata={"visual_point": [30, 40], "blocking_overlay": True, "overlay_name": name,
+ "overlay_kind": "obstruction", "overlay_bounds": [10, 20, 100, 100]})]
+ context = controller.finalize(space, [], memory.active_stage, memory, tree, "S1:O0")
+ controls.append(controller.control)
+ controller.control["ground_calls"] = 2
+ view = controller.choice_context({**execution_context(memory), **context})
+ assert "Restore input access" in view["objective"]
+ assert view["inputs"] == [] and "Form completed" not in view["exit_condition"]
+ assert controls[0] is controls[1] is controls[2]
+ assert controls[-1]["ground_calls"] == 2
+ controller.finalize(goal_candidates(build_space([]), [], tree, []), [],
+ memory.active_stage, memory, tree, "S1:O2")
+ assert memory.active_blocker["status"] == "resolved"
+ assert memory.active_blocker["resolution_evidence_refs"] == ["S1:O2"]
+
+
+def test_required_task_dialog_is_not_dismissed_after_failure():
+ controller, memory, _ = setup_controller()
+ tree = ''
+ space = goal_candidates(build_space([]), [], tree, [])
+ space.target_heads["visual"] = [ActionCandidate("save", "click", "Save", source="visual_grounder",
+ metadata={"visual_point": [30, 40], "blocking_overlay": True, "overlay_name": "Save",
+ "overlay_kind": "task_dialog", "overlay_bounds": [10, 20, 100, 100]})]
+ controller.finalize(space, [], memory.active_stage, memory, tree)
+ controller.control["recovery_required"] = True
+ context = controller.finalize(space, [], memory.active_stage, memory, tree)
+ assert not context["scope_recovery"]
diff --git a/tests/harness/test_goal_delivery_loop.py b/tests/harness/test_goal_delivery_loop.py
new file mode 100644
index 0000000..0a23800
--- /dev/null
+++ b/tests/harness/test_goal_delivery_loop.py
@@ -0,0 +1,187 @@
+"""Synthetic regressions for the execution/observation/delivery handoffs; not benchmark scores."""
+
+import json
+from dataclasses import replace
+from unittest.mock import Mock
+
+import pytest
+
+from tandem.gui.goal_candidates import (
+ foreground_tree,
+ goal_candidates,
+ modal_frontier,
+ portal_needs_grounding,
+)
+from tandem.gui.interaction import GuiInteractionController, target_effect
+from tandem.gui.semantics import enrich_targets
+from tandem.memory.context import execution_context, planning_context, verification_context
+from tandem.memory.state import TaskMemory
+from tandem.runtime.goal_stage import ObservationRequired
+from tandem.verification.verifier import verify_observation
+
+from .conftest import build_space, build_targets, cli_receipt, make_observation
+from .test_goal_memory import session
+from .test_goal_stage import plan, requirements, result
+
+PORTAL = '''
+
+
+'''
+
+
+def test_coordinate_free_portal_is_retained_for_visual_confirmation():
+ tree = foreground_tree(PORTAL)
+ assert "_Open Open" in tree and "Submit" in tree
+ assert portal_needs_grounding(tree) == "Open Files"
+ # Coordinate-free nodes are not turned into invented click coordinates.
+ assert not [t for t in build_targets(tree) if t.name == "_Open Open"]
+
+
+def test_coordinate_free_portal_requests_grounder_despite_rich_background(tmp_path):
+ s = session()
+ s.begin("Upload the document", requirements())
+ s.goal.memory.start(plan(), 0)
+ s.components.grounder = Mock(return_value=[])
+ s.act(make_observation(tmp_path, 0, tree_xml=PORTAL))
+ s.components.grounder.assert_called_once()
+
+
+def test_native_dialog_keeps_focus_and_selection_keys():
+ tree = ''
+ space = goal_candidates(build_space([]), [], tree, [])
+ modal_frontier(space, [], tree)
+ offered = [c.metadata.get("key_sequence") for c in space.candidates() if c.operation == "keypress"]
+ assert [["f6"]] in offered and [["space"]] in offered
+ assert [["ctrl", "t"]] not in offered # Do not navigate behind an active file dialog.
+
+
+def test_foreground_window_missing_from_a11y_cannot_offer_background_controls(tmp_path):
+ tree = '''
+ '''
+ assert not enrich_targets(build_targets(tree), tree, (1920, 1080))
+ s = session()
+ s.begin("Open the reference", requirements())
+ s.goal.memory.start(plan(), 0)
+ s.components.grounder = Mock(return_value=[])
+ s.act(make_observation(tmp_path, 0, tree_xml=tree))
+ s.components.grounder.assert_called_once()
+
+
+def test_field_hint_does_not_replace_preservation_goal():
+ controller = GuiInteractionController()
+ context = {"goal": "Read the reference in a NEW tab and preserve the filled form",
+ "completion_criteria": "Reference read and form retained", "inputs": [],
+ "next_intent": "Type the reference URL into the address bar", "constraints": [],
+ "method": "Open a new tab first", "guidance": "Never replace the existing form tab"}
+ view = controller.choice_context(context)
+ assert view["objective"] == context["goal"]
+ assert view["next_action_hint"] == context["next_intent"]
+ assert view["guidance"] == context["guidance"]
+
+
+def test_opened_document_consumes_its_now_absent_path_entry():
+ memory = TaskMemory("Read reference", requirements())
+ value = {"label": "Reference path", "target_hint": "Enter location or URL", "text": "/tmp/reference.png",
+ "input_mode": "replace", "commit": "enter"}
+ memory.start({**plan(), "purpose": "prepare", "inputs": [value]}, 0)
+ controller = GuiInteractionController()
+ controller.stage_id = memory.active_stage["id"]
+ key = json.dumps([value["target_hint"].casefold(), value["text"]], ensure_ascii=False)
+ controller.fields[key] = {"target": value["target_hint"], "expected_value": value["text"],
+ "phase": "submitted_unconfirmed", "commit": "enter"}
+ tree = ''
+ space = goal_candidates(build_space([]), [], tree, [value])
+ context = controller.finalize(space, [], memory.active_stage, memory, tree)
+ assert context["unbound_inputs"] == [] and controller.fields[key]["phase"] == "consumed"
+ assert controller.choice_context({**execution_context(memory), **context})["inputs"] == []
+
+
+def test_unrelated_focused_field_cannot_disprove_a_visual_input():
+ target = replace(build_targets()[0], name="Address and search bar", value="old", focused=True)
+ pending = {"operation": "type_text", "field_label": "Enter location or URL",
+ "expected_text": "/tmp/reference.png", "input_mode": "replace"}
+ assert target_effect(pending, [target]) == "unknown"
+
+
+def test_unpublished_working_copy_is_not_saved_delivery_then_can_be_published():
+ memory = TaskMemory("Update the board deck", requirements())
+ memory.start(plan(), 0)
+ first = result(artifacts=[{"target_path": "/tmp/working.pptx", "working_path": "/tmp/working.pptx",
+ "save_state": "saved", "evidence_refs": ["A2"]}])
+ memory.apply(first, {"A2": {}})
+ assert memory.work_items["R1:unit:1"]["status"] == "pending"
+ assert first["delivery_state"] == "implemented" and not first["unit_met"]
+ assert planning_context(memory)["pending_publications"]
+ memory.start({**plan(), "purpose": "save"}, 2)
+ merely_open = result(evidence_refs=["A3"])
+ memory.apply(merely_open, {"A3": {}})
+ assert not merely_open["unit_met"] # Opening the temporary copy does not publish it.
+ memory.start({**plan(), "purpose": "save"}, 3)
+ final = result(evidence_refs=["A3"], artifacts=[{
+ "target_path": "/home/user/Desktop/board.pptx", "working_path": "/home/user/Desktop/board.pptx",
+ "save_state": "saved", "replaces_working_path": "/tmp/working.pptx", "evidence_refs": ["A3"]}])
+ memory.apply(final, {"A3": {}})
+ assert memory.work_items["R1:unit:1"]["status"] == "done"
+ assert not planning_context(memory)["pending_publications"]
+
+
+def test_rollback_revokes_requirement_status_not_confirmed_source_facts():
+ reqs = requirements()
+ reqs[0]["units"].append("Another section")
+ reqs[0]["unit_fact_keys"].append([])
+ memory = TaskMemory("Complete form", reqs)
+ memory.start(plan(), 0)
+ fact = {"key": "source", "value": "Stable source", "status": "confirmed", "scope": "task",
+ "source_ref": "input.pdf", "evidence_refs": ["A2"]}
+ memory.apply(result(facts=[fact], requirement_checks=[{
+ "id": "R1", "status": "met", "evidence_refs": ["A2"]}]), {"A2": {}})
+ memory.start({**plan(), "work_item_id": "R1:unit:2", "purpose": "prepare"}, 2)
+ memory.apply(result(unit_met=False, delivery_state="none", evidence_refs=["A3"],
+ invalidated_work_item_ids=["R1:unit:1"]), {"A3": {}})
+ assert memory.requirements[0]["status"] == "uncertain"
+ assert memory.work_items["R1:unit:1"]["status"] == "pending"
+ assert memory.facts["source"]["value"] == "Stable source"
+
+
+def test_real_dispatch_resets_terminal_rejection_counter(tmp_path):
+ s = session()
+ s.begin("Edit", requirements())
+ s.goal.memory.start({**plan(), "route": "cli", "command": "true"}, 0)
+ s.consecutive_no_action = 2
+ assert s.act(make_observation(tmp_path, 0)).kind == "cli"
+ with pytest.raises(ObservationRequired, match="stage_verification_returned"):
+ s.act(make_observation(tmp_path, 1, receipt=cli_receipt()))
+ assert s.consecutive_no_action == 0
+
+
+def test_terminal_verifier_receives_independent_stage_ledger(tmp_path):
+ memory = TaskMemory("Finish", requirements())
+ memory.start(plan(), 0)
+ memory.apply(result(), {"A2": {}})
+ client = Mock()
+
+ def request(messages, validate, **kwargs):
+ context = json.loads(messages[1]["content"][0]["text"])
+ assert context["verified_task_memory"]["work_items"][0]["status"] == "done"
+ assert "simultaneously" in messages[0]["content"]
+ return {"outcome": "achieved", "task_status": "completed", "evidence": "stage A2",
+ "path_input_focused": False, "state_changed": False, "requirement_checks": []}
+
+ client.json_request.side_effect = request
+ make_observation(tmp_path, 0)
+ verify_observation(client, "Finish", tmp_path / "step_00.png", terminal_claim={"done": True},
+ task_memory=verification_context(memory))
+ client.json_request.assert_called_once()
+
+
+def test_planner_gets_actual_wall_budget(tmp_path):
+ s = session()
+ s.begin("Edit", requirements())
+ s.components.planner = Mock(return_value=plan())
+ with pytest.raises(ObservationRequired, match="planner_returned"):
+ s.act(make_observation(tmp_path, 0, wall=321))
+ assert s.components.planner.call_args.kwargs["remaining_seconds"] == 321
diff --git a/tests/harness/test_goal_execution.py b/tests/harness/test_goal_execution.py
new file mode 100644
index 0000000..c64db49
--- /dev/null
+++ b/tests/harness/test_goal_execution.py
@@ -0,0 +1,236 @@
+"""Synthetic regressions derived from real failure patterns; not benchmark acceptance."""
+
+from pathlib import Path
+from unittest.mock import Mock
+
+import pytest
+
+from tandem.gui.compile import gui_action
+from tandem.gui.goal_candidates import foreground_tree, goal_candidates
+from tandem.gui.targets import A11yTargetBuilder
+from tandem.memory.context import planning_context
+from tandem.models.ranking import Ranking
+from tandem.planning.goal_planner import goal_error, plan_goal
+from tandem.runtime.goal_stage import (
+ ObservationRequired,
+ _action_key,
+ _cycle_actions,
+ _loop_state,
+ _target_effect,
+)
+from tandem.verification.goal_verifier import result_error
+
+from .conftest import A11Y_XML, build_space, build_targets, gui_receipt, make_observation
+from .test_goal_memory import fact, session
+from .test_goal_stage import plan, requirements, result
+
+
+def _space(tree, inputs=()):
+ tree = foreground_tree(tree)
+ targets = A11yTargetBuilder().build(tree, "test", 128, True, True, filter_occluded=True)
+ return targets, goal_candidates(build_space(targets), targets, tree, inputs)
+
+
+def test_receipt_path_detects_a_b_a_b_without_manually_seeding_attempts(tmp_path):
+ s = session()
+ s.begin("Edit", requirements())
+ s.goal.memory.start(plan(), 0)
+ chosen = []
+
+ def rank(**kw):
+ candidates = kw["action_space"].candidates()
+ toggles = [c for c in candidates if c.operation == "click" and "启用" in c.description]
+ if len(chosen) == 4:
+ assert not toggles
+ assert kw["action_space"].controls["return_to_planner"]
+ winner = next(c for c in candidates if c.id == "KEYPRESS_ESCAPE")
+ else:
+ winner = toggles[0]
+ chosen.append(winner)
+ return Ranking(winner, [], [], 0, 0, 1, True)
+
+ s.components.selector.rank = rank
+ for step in range(4):
+ tree = A11Y_XML.replace('checked="false"', f'checked="{str(step % 2 == 1).lower()}"')
+ s.act(make_observation(tmp_path, step, tree_xml=tree,
+ receipt=gui_receipt() if step else None))
+ with pytest.raises(ObservationRequired, match="stage_verification_returned"):
+ s.act(make_observation(tmp_path, 4, tree_xml=A11Y_XML, receipt=gui_receipt()))
+ assert len(chosen) == 4 # The return cycle ends before offering another ineffective toggle.
+ assert any(e.get("end_reason") == "execution_checkpoint_stalled" for e in s.events)
+ assert s.goal.memory.execution_constraints
+ assert planning_context(s.goal.memory)["execution_constraints"]
+ assert s.goal.attempts[0]["state"] == s.goal.attempts[2]["state"]
+
+
+def test_duplicate_popup_nodes_do_not_remove_their_own_close_button():
+ popup = (''
+ ''
+ '')
+ tree = (''
+ ''
+ + popup * 2 + '')
+ targets = A11yTargetBuilder().build(tree, "test", filter_occluded=True)
+ assert [t.name for t in targets].count("Close") == 2
+ assert "PDF" not in [t.name for t in targets]
+ assert "Not covered" in [t.name for t in targets]
+
+
+def test_explicit_popup_menu_excludes_the_covered_media_button():
+ tree = (''
+ '')
+ ts, space = _space(tree)
+ assert "More" not in [t.name for t in ts]
+ assert any("Exit picture-in-picture" in c.description for c in space.target_heads["click"])
+
+
+def test_browser_memory_and_playback_badges_do_not_create_new_loop_states():
+ def state(name):
+ tree = ('')
+ return _loop_state(build_targets(tree), tree)
+
+ assert state("Video - Memory usage - 47.1 MB") == state(
+ "Video – Audio playing - Memory usage - 108 MB")
+ assert state("Different page - Memory usage - 108 MB") != state("Video - Memory usage - 47.1 MB")
+
+
+def test_four_state_cycle_is_detected_but_continuous_scroll_is_not():
+ states = ["closed", "menu", "pip", "pip-menu"]
+ attempts = [{"work": "w", "state": state, "key": ("click", "menu", "")}
+ for state in states * 2]
+ assert _cycle_actions(attempts, "closed") == {("click", "menu", "")}
+ assert not _cycle_actions([{"state": "page", "key": ("scroll", "down", "")}] * 8, "page")
+
+
+def test_overview_excludes_stale_app_and_keeps_unnamed_search_then_typing():
+ tree = (''
+ ''
+ ''
+ ''
+ '')
+ targets, space = _space(tree)
+ assert "Play" not in [t.name for t in targets]
+ assert any("Desktop search" in c.description for c in space.target_heads["click"])
+ assert not space.target_heads["type_text"] # Must focus the observed field first.
+ _, space = _space(tree.replace('focused="false"', 'focused="true"'), [
+ {"label": "Application", "target_hint": "Desktop search", "text": "Shotcut",
+ "input_mode": "replace"}])
+ assert gui_action(space.target_heads["type_text"][0], None).parameters["text"] == "Shotcut"
+
+
+def test_hidden_overview_does_not_take_over_the_foreground():
+ tree = (''
+ ''
+ '')
+ assert 'name="app"' in foreground_tree(tree)
+
+
+def test_slider_has_bounded_distinct_parameters_and_checks_the_intended_value():
+ tree = ('')
+ targets, space = _space(tree)
+ choices = space.target_heads["set_slider"]
+ assert len(choices) == 4 and not space.target_heads["click"]
+ assert len({_action_key(c, targets) for c in choices}) == 4
+ winner = next(c for c in choices if c.metadata["fraction"] == 0.75)
+ assert gui_action(winner, targets[0]).parameters["x"] == 850
+ pending = {"operation": "set_slider", "target_semantic": targets[0].semantic_key, "fraction": 0.75}
+ assert _target_effect(pending, targets) == "not_met"
+ updated, _ = _space(tree.replace('value="50"', 'value="75"'))
+ assert _target_effect(pending, updated) == "met"
+
+
+def test_new_source_fact_does_not_clear_a_failed_gui_action(tmp_path):
+ s = session()
+ s.begin("Edit", requirements())
+ s.goal.memory.start({**plan(), "purpose": "prepare"}, 0)
+ s.goal.memory.source_keys.add("quota")
+ s.goal.failed_actions = {("R1:unit:1", "state", "candidate"): 2}
+ s.goal.memory.execution_constraints = [{"work_item_id": "R1:unit:1"}]
+ s.components.stage_verifier = Mock(return_value=result(
+ unit_met=False, stage_outcome="not_met", delivery_state="none", evidence_refs=["S1:O2"],
+ facts=[fact(evidence_refs=["S1:O2"])]))
+ s.goal.end_reason = "returned_to_planner"
+ with pytest.raises(ObservationRequired):
+ s.goal._finish(s, make_observation(tmp_path, 2))
+ assert s.goal.memory.facts["quota"]["value"] == "1400000 USD"
+ assert s.goal.failed_actions and s.goal.memory.execution_constraints
+
+
+def test_planner_application_handoff_supplies_actual_typable_input(tmp_path):
+ s = session()
+ s.begin("Open application", requirements())
+
+ class Client:
+ def json_request(self, messages, validate, **kwargs):
+ value = {**plan(), "target_application": "Shotcut"}
+ assert validate(value)
+ return value
+
+ shot = tmp_path / "screen.png"
+ shot.write_bytes(b"synthetic")
+ value = plan_goal(Client(), "Open application", shot, [], [], "", memory=s.goal.memory)
+ assert value["inputs"][-1]["text"] == "Shotcut"
+
+
+def test_contract_repair_names_the_actual_field_or_semantic_failure():
+ s = session()
+ s.begin("Edit", requirements())
+ assert goal_error({**plan(), "work_item_id": "R9"}, s.goal.memory, True).startswith("work_item_id")
+ assert goal_error({**plan(), "inputs": [{}]}, s.goal.memory, True).startswith("inputs[0]")
+ assert goal_error({**plan(), "work_item_id": {}}, s.goal.memory, True).startswith("work_item_id")
+ missing = plan()
+ missing.pop("command")
+ assert goal_error(missing, s.goal.memory, True).startswith("command")
+ item = s.goal.memory.work_items["R1:unit:1"]
+ item.update(stalled_stages=2, last_attempt={"route": "cli", "command": "same"})
+ assert "repeats" in goal_error({**plan(), "route": "cli", "command": "same"}, s.goal.memory, True)
+ assert result_error(result(facts=[{**fact(), "value": {}}])).startswith("facts[0].value")
+ assert result_error(result(stage_outcome=[])).startswith("stage_outcome")
+ assert result_error(result(materials=[{"source_ref": "x", "summary": "x", "evidence_refs": []}]
+ )).startswith("materials[0].covered_parts")
+
+
+def test_planner_repair_preserves_punctuation_in_memory_keys(tmp_path):
+ s = session()
+ s.begin("Edit the supplied deck", requirements())
+ exact = "deck.pptx_open_document"
+ s.goal.memory.facts[exact] = fact(key=exact, scope="task")
+
+ class Client:
+ def json_request(self, messages, validate, **kwargs):
+ assert not validate({**plan(), "input_fact_keys": ["deck_pptx_open_document"]})
+ assert "copy keys EXACTLY" in messages[-1]["content"]
+ assert exact in messages[-1]["content"]
+ value = {**plan(), "input_fact_keys": [exact]}
+ assert validate(value)
+ return value
+
+ observation = make_observation(tmp_path, 0)
+ value = plan_goal(Client(), "Edit", Path(observation.screenshot.path), [], [], "",
+ memory=s.goal.memory)
+ assert value["input_fact_keys"] == [exact]
+
+
+def test_gui_plan_rejects_multiple_values_for_one_field_but_allows_distinct_fields():
+ s = session()
+ s.begin("Read the references", requirements())
+ inputs = [{"label": str(i), "target_hint": "Address and search bar", "text": f"https://example/{i}",
+ "input_mode": "replace"} for i in range(2)]
+ assert "conflicting texts" in goal_error({**plan(), "inputs": inputs}, s.goal.memory, True)
+ inputs[1]["target_hint"] = "Search"
+ assert not goal_error({**plan(), "inputs": inputs}, s.goal.memory, True)
+
+
+def test_different_fields_and_changed_pages_are_not_the_same_failed_action():
+ first = build_targets()
+ second = build_targets(A11Y_XML.replace("确定", "Next page"))
+ one = build_space(first).target_heads["click"][0]
+ two = build_space(second).target_heads["click"][0]
+ assert _action_key(one, first) != _action_key(two, second)
diff --git a/tests/harness/test_goal_integration.py b/tests/harness/test_goal_integration.py
new file mode 100644
index 0000000..582eed7
--- /dev/null
+++ b/tests/harness/test_goal_integration.py
@@ -0,0 +1,328 @@
+"""Synthetic compatibility tests against main's semantic pipeline and JEV recovery."""
+
+import base64
+import io
+import json
+import urllib.error
+from dataclasses import replace
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+import pytest
+from PIL import Image
+
+from tandem.config import SessionConfig
+from tandem.gui.candidates import ActionCandidate
+from tandem.memory.state import TaskMemory
+from tandem.models.jev import JevOmniPolicy
+from tandem.models.ranking import Ranking
+from tandem.models.slow import SlowModelClient
+from tandem.protocol.models import BrowserSnapshot
+from tandem.runtime.goal_stage import ObservationRequired
+from tandem.verification.goal_verifier import verify_goal
+
+from .conftest import build_space, build_targets, gui_receipt, make_observation
+from .test_goal_memory import session
+from .test_goal_stage import plan, requirements
+from .test_selector_compile import ScriptedOpener
+from .test_slow_transport import _Response
+
+
+def test_goal_passes_cdp_enriched_semantics_to_planner_and_selector(tmp_path):
+ s = session()
+ s.begin("Enter the receipt date", requirements())
+ tree = (''
+ ''
+ ''
+ '')
+ observation = make_observation(tmp_path, 0, tree_xml=tree)
+ snapshot = tmp_path / "browser.json"
+ snapshot.write_text(json.dumps({"elements": [{
+ "index": 1, "role": "entry", "name": "Receipt Date", "description": "dd/mm/yyyy",
+ "value": "", "x": 250, "y": 293, "width": 218, "height": 18,
+ "label_source": "dom_label", "operations": ["click", "type_text"],
+ }]}))
+ observation = replace(observation, browser_snapshot=BrowserSnapshot(
+ present=True, source="cdp", path=str(snapshot), node_count=1))
+
+ def planner(*args, **kwargs):
+ target = next(t for t in args[2] if t.name == "Receipt Date")
+ assert target.source == "dom" and target.scope == "web_content"
+ return {**plan(), "inputs": [{"label": "Receipt Date", "target_hint": "Receipt Date",
+ "text": "11/12/2025", "input_mode": "replace"}]}
+
+ s.components.planner = planner
+ with pytest.raises(ObservationRequired, match="planner_returned"):
+ s.act(observation)
+
+ def rank(**kwargs):
+ winner = kwargs["action_space"].target_heads["type_text"][0]
+ target = next(t for t in kwargs["targets"] if t.id == winner.target_id)
+ assert target.name == "Receipt Date" and target.description == "dd/mm/yyyy"
+ return Ranking(winner, [], [], 0, 0, 1, True)
+
+ s.components.selector.rank = rank
+ action = s.act(observation)
+ assert action.action_type == "type" and action.parameters["click"] == [359, 302]
+
+
+def test_joint_goal_choices_retain_main_endpoint_recovery(monkeypatch):
+ monkeypatch.setattr("tandem.models.jev.time.sleep", lambda seconds: None)
+ targets = build_targets()
+ space = build_space(targets)
+ space.target_heads = {"click": space.target_heads["click"][:1]}
+ space.controls = {"done": space.controls["done"]}
+ click, done = space.candidates()
+ opener = ScriptedOpener([
+ urllib.error.URLError("old endpoint unavailable"), {"status": "ok"},
+ {"scores": [0.9, 0.1], "used_image": True}, {"status": "ok"},
+ {"scores": [0.1, 0.9], "used_image": True},
+ ])
+ endpoints = iter(["http://old", "http://new", "http://new"])
+ policy = JevOmniPolicy("http://old", opener=opener, joint_actions=True,
+ endpoint_provider=lambda: next(endpoints))
+ ranking = policy.rank(stage_goal="Goal", intent="Intent", active_window="Chrome",
+ targets=targets, action_space=space, recent_actions=[])
+ assert ranking.operation_ranked[0]["operation"] == click.operation
+ assert ranking.winner is done and policy.endpoint == "http://new"
+ assert opener.requests[-2:] == ["http://new/", "http://new/rank"]
+
+
+def test_goal_image_budget_and_verify_override_do_not_change_step_transport(tmp_path):
+ encoded = io.BytesIO()
+ Image.new("RGB", (100, 100), "white").save(encoded, format="PNG")
+ image_url = "data:image/png;base64," + base64.b64encode(encoded.getvalue()).decode()
+ for mode in ("step", "goal"):
+ config = SessionConfig(execution_mode=mode, slow_max_tokens=32768,
+ slow_verify_max_tokens=8192, slow_reasoning_effort="max",
+ slow_verify_reasoning_effort="low")
+ client = SlowModelClient({"base_url": "https://test.invalid"}, config, tmp_path / "audit.jsonl")
+ messages = [{"role": "user", "content": [{"type": "image_url",
+ "image_url": {"url": image_url}}]}]
+ with patch("urllib.request.urlopen", return_value=_Response()) as opener:
+ client.json_request(messages, bool, role="verify_stage")
+ payload = json.loads(opener.call_args.args[0].data)
+ sent_url = payload["messages"][0]["content"][0]["image_url"]["url"]
+ assert sent_url.startswith("data:image/jpeg" if mode == "goal" else "data:image/png")
+ assert payload["max_tokens"] == 8192 and payload["reasoning_effort"] == "low"
+
+
+def test_goal_memory_and_stage_verifier_are_replaceable():
+ s = session()
+ factory = Mock(return_value=TaskMemory("Edit", requirements()))
+ s.components.memory_factory = factory
+ s.components.stage_verifier = Mock()
+ s.begin("Edit", requirements())
+ factory.assert_called_once_with("Edit", requirements())
+ assert s.goal.memory is factory.return_value
+
+
+def _shell_observation(directory, *, clock="20:00", window="Editor", color="white", step=0):
+ directory.mkdir()
+ tree = f'''
+
+
+ {clock}
+
+ '''
+ observation = make_observation(directory, step, tree_xml=tree)
+ Image.new("RGB", (64, 64), color).save(observation.screenshot.path)
+ return observation
+
+
+def _visual(point=(20, 40)):
+ return ActionCandidate("visual", "click", "Visible application launcher", source="visual_grounder",
+ metadata={"visual_point": list(point)})
+
+
+def test_clock_change_after_grounding_does_not_starve_jev(tmp_path):
+ s = session()
+ s.begin("Open the application", requirements())
+ s.components.grounder = Mock(return_value=[_visual()])
+
+ def rank(**kwargs):
+ assert "Activities" in [t.name for t in kwargs["targets"]] # Still actionable.
+ winner = kwargs["action_space"].target_heads["visual"][0]
+ return Ranking(winner, [], [], 0, 0, 1, True)
+
+ s.components.selector.rank = Mock(side_effect=rank)
+ before = _shell_observation(tmp_path / "before")
+ with pytest.raises(ObservationRequired, match="planner_returned"):
+ s.act(before)
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(before)
+ refreshed = _shell_observation(tmp_path / "after", clock="20:02")
+ action = s.act(refreshed)
+ assert action.parameters == {"x": 20, "y": 40}
+ s.components.grounder.assert_called_once()
+ s.components.selector.rank.assert_called_once()
+ assert s.goal.visual and s.goal.visual_frame # Keep valid targets until the scene actually changes.
+
+
+@pytest.mark.parametrize("change", ["window", "pixels", "size"])
+def test_changed_scene_discards_pending_visual_coordinates(tmp_path, change):
+ s = session()
+ s.begin("Open the application", requirements())
+ s.components.grounder = Mock(side_effect=[[_visual()], [_visual((30, 45))]])
+ s.components.selector.rank = Mock(side_effect=lambda **kw: Ranking(
+ kw["action_space"].target_heads["visual"][0], [], [], 0, 0, 1, True))
+ before = _shell_observation(tmp_path / "before")
+ with pytest.raises(ObservationRequired):
+ s.act(before)
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(before)
+ after = _shell_observation(tmp_path / "after", window="Viewer" if change == "window" else "Editor",
+ color="black" if change == "pixels" else "white")
+ if change == "size":
+ Image.new("RGB", (80, 64), "white").save(after.screenshot.path)
+ # A changed scene needs one new grounding, never the old point or a replayed GUI action.
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(after)
+ s.components.selector.rank.assert_not_called()
+ assert s.act(after).parameters == {"x": 30, "y": 45}
+ assert s.components.grounder.call_count == 2
+
+
+@pytest.mark.parametrize("budget", ["calls", "elapsed"])
+def test_exhausted_grounding_keeps_existing_legal_actions_available(tmp_path, monkeypatch, budget):
+ s = session()
+ s.begin("Open the application", requirements())
+ s.components.grounder = Mock(return_value=[_visual()])
+ s.components.selector.rank = Mock(side_effect=lambda **kw: Ranking(
+ next(c for c in kw["action_space"].candidates() if c.operation == "keypress"),
+ [], [], 0, 0, 1, True))
+ client = Mock()
+ s.components.stage_verifier = lambda *a, **kw: verify_goal(client, *a, **kw)
+ now = [100.0]
+ monkeypatch.setattr("tandem.runtime.goal_stage.time.monotonic", lambda: now[0])
+
+ def ground(*args):
+ if budget == "elapsed":
+ now[0] += s.config.goal_recovery_max_seconds
+ return [_visual()]
+
+ s.components.grounder.side_effect = ground
+ first = _shell_observation(tmp_path / "first")
+ with pytest.raises(ObservationRequired):
+ s.act(first)
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(first)
+ if budget == "calls":
+ second = _shell_observation(tmp_path / "second", window="Second")
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(second)
+ changed = _shell_observation(tmp_path / "third", window="Third")
+ assert s.act(changed).kind == "gui"
+ assert s.components.grounder.call_count == (1 if budget == "elapsed" else 2)
+ s.components.selector.rank.assert_called_once()
+ client.json_request.assert_not_called()
+ assert s.goal.memory.active_stage is not None
+ assert s.goal.memory.active_blocker["kind"] == "capability_exhausted"
+ assert not any(e.get("route") == "stage_verified" for e in s.events)
+ assert not any(c.operation == "visual_fallback" for c in
+ s.components.selector.rank.call_args.kwargs["action_space"].candidates())
+
+
+def test_clock_only_changes_do_not_reset_failed_action_memory(tmp_path):
+ s = session()
+ s.begin("Open the application", requirements())
+ s.components.selector.rank = Mock(side_effect=lambda **kw: Ranking(
+ next(c for c in kw["action_space"].candidates() if c.operation == "keypress"),
+ [], [], 0, 0, 1, True))
+ first = _shell_observation(tmp_path / "first")
+ with pytest.raises(ObservationRequired):
+ s.act(first)
+ s.act(first)
+ refreshed = replace(_shell_observation(tmp_path / "after", clock="20:03", step=1),
+ last_receipt=gui_receipt())
+ s.act(refreshed)
+ assert s.goal.no_effect == 1
+ assert len(s.goal.failed_actions) == 1 and not s.goal.history[-1]["relevant_change"]
+
+
+def test_consumed_visual_cache_expires_when_pixels_change_without_a11y_change(tmp_path):
+ s = session()
+ s.begin("Edit the current slide", requirements())
+ s.components.grounder = Mock(side_effect=[[_visual()], [_visual((30, 45))]])
+ s.components.selector.rank = Mock(side_effect=lambda **kw: Ranking(
+ kw["action_space"].target_heads["visual"][0], [], [], 0, 0, 1, True))
+ before = _shell_observation(tmp_path / "before")
+ with pytest.raises(ObservationRequired, match="planner_returned"):
+ s.act(before)
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(before)
+ assert s.act(before).parameters == {"x": 20, "y": 40}
+ after = replace(_shell_observation(tmp_path / "after", color="black", step=1),
+ last_receipt=gui_receipt())
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(after)
+ assert s.components.grounder.call_count == 2
+ assert any(e.get("route") == "recovery_cache_invalidated" for e in s.events)
+ assert s.act(after).parameters == {"x": 30, "y": 45}
+
+
+def test_same_scene_new_stage_does_not_inherit_old_grounding_suppression(tmp_path):
+ s = session()
+ s.begin("Edit two portions of the same slide", requirements())
+ s.components.planner = lambda *a, **kw: {**plan(), "max_actions": 1}
+ s.components.grounder = Mock(return_value=[_visual()])
+ s.components.selector.rank = Mock(side_effect=lambda **kw: Ranking(
+ kw["action_space"].target_heads["visual"][0], [], [], 0, 0, 1, True))
+ before = _shell_observation(tmp_path / "before")
+ with pytest.raises(ObservationRequired):
+ s.act(before)
+ with pytest.raises(ObservationRequired):
+ s.act(before)
+ s.act(before)
+ s.goal.recovery_calls = s.config.goal_recovery_max_calls
+ s.goal.recovery_seconds = s.config.goal_recovery_max_seconds
+ after = replace(_shell_observation(tmp_path / "after", step=1), last_receipt=gui_receipt())
+ with pytest.raises(ObservationRequired, match="stage_verification_returned"):
+ s.act(after)
+ with pytest.raises(ObservationRequired, match="planner_returned"):
+ s.act(after)
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(after)
+ assert s.components.grounder.call_count == 2
+ assert s.goal.recovery_calls == 1 and s.goal.recovery_seconds < 1
+
+
+def test_waiting_outside_grounder_does_not_consume_visual_recovery_time(tmp_path, monkeypatch):
+ s = session()
+ s.begin("Open the application", requirements())
+ now = [100.0]
+ monkeypatch.setattr("tandem.runtime.goal_stage.time.monotonic", lambda: now[0])
+ s.components.grounder = Mock(return_value=[_visual()])
+ before = _shell_observation(tmp_path / "before")
+ with pytest.raises(ObservationRequired):
+ s.act(before)
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(before)
+ now[0] += 10000
+ after = _shell_observation(tmp_path / "after", window="Another application")
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(after)
+ assert s.goal.recovery_calls == 2 and s.goal.recovery_seconds == 0
+
+
+def test_shell_menu_toggle_is_not_task_progress_or_a_recovery_budget_reset(tmp_path):
+ s = session()
+ s.begin("Import an image into the editor", requirements())
+ s.components.selector.rank = Mock(side_effect=lambda **kw: Ranking(
+ next(c for c in kw["action_space"].candidates() if c.operation == "click"),
+ [], [], 0, 0, 1, True))
+ before = _shell_observation(tmp_path / "before")
+ with pytest.raises(ObservationRequired):
+ s.act(before)
+ s.act(before)
+ s.goal.recovery_calls = 1
+ after = _shell_observation(tmp_path / "after", step=1)
+ path = after.accessibility_tree.path
+ tree = Path(path).read_text().replace('',
+ ''
+ '')
+ Path(path).write_text(tree)
+ s.act(replace(after, last_receipt=gui_receipt()))
+ assert s.goal.no_effect == 1 and s.goal.recovery_calls == 1
+ assert s.goal.history[-1]["observed_change"]
+ assert not s.goal.history[-1]["relevant_change"]
diff --git a/tests/harness/test_goal_interaction.py b/tests/harness/test_goal_interaction.py
new file mode 100644
index 0000000..a86e294
--- /dev/null
+++ b/tests/harness/test_goal_interaction.py
@@ -0,0 +1,344 @@
+"""Synthetic control/acceptance regressions; not real GUI or benchmark acceptance."""
+
+import json
+from dataclasses import replace
+from pathlib import Path
+from unittest.mock import Mock
+
+import pytest
+from PIL import Image, ImageDraw
+
+from tandem.components import build_default
+from tandem.config import SessionConfig
+from tandem.gui.candidates import ActionCandidate
+from tandem.gui.compile import gui_action
+from tandem.gui.goal_candidates import _input_target, application_mismatch, goal_candidates, input_frontier
+from tandem.gui.interaction import GuiInteractionController, visual_changed
+from tandem.memory.context import planning_context
+from tandem.memory.state import TaskMemory
+from tandem.models.ranking import Ranking
+from tandem.planning.goal_planner import goal_error
+from tandem.runtime.goal_stage import ObservationRequired, _action_key
+from tandem.verification.goal_verifier import result_error, verify_goal
+
+from .conftest import build_space, gui_receipt, make_observation
+from .test_goal_execution import _space
+from .test_goal_integration import _shell_observation
+from .test_goal_memory import session
+from .test_goal_progress import FORM
+from .test_goal_stage import plan, requirements, result
+
+
+def _input(text="laptop.pptx", hint="File name", **kw):
+ return {"label": hint, "target_hint": hint, "text": text, "input_mode": "replace", **kw}
+
+
+def _visual(value, point=(20, 45)):
+ return ActionCandidate("visual-input", "type_text", "Type into the observed field",
+ source="visual_grounder", metadata={"visual_point": list(point), "visual_bounds": [10, 40, 30, 10],
+ "target_hint": value["target_hint"], "target_label": value["target_hint"],
+ "field_label": value["target_hint"], "text": value["text"], "input_mode": "replace"})
+
+
+def _checked_result(**kw):
+ v = result(progress_kind="content", observed_errors=[], **kw)
+ v["unit_assessment"] = {"work_item_id": "R1:unit:1", "status": "met" if v["unit_met"] else "not_met",
+ "evidence_refs": ["A2"], "remaining": []}
+ return v
+
+
+def test_interaction_factory_is_injectable():
+ built = build_default(client=Mock(), endpoint="http://unused",
+ config=SessionConfig(execution_mode="goal"))
+ assert built.interaction_factory is GuiInteractionController
+ s = session()
+ controller = GuiInteractionController()
+ s.components.interaction_factory = Mock(return_value=controller)
+ s.begin("Edit", requirements())
+ assert s.goal.interaction is controller
+
+
+def test_tab_misselection_preserves_visual_input_and_specific_intent(tmp_path):
+ s = session()
+ s.begin("Save laptop.pptx", requirements())
+ value = _input()
+ s.goal.memory.start({**plan(), "inputs": [value]}, 0)
+ choice = _visual(value)
+ choice.metadata.update(blocking_overlay=True, overlay_name="Save As", overlay_bounds=[5, 30, 50, 30])
+ s.components.grounder = Mock(return_value=[choice])
+ selected = []
+
+ def rank(**kw):
+ assert any(c.operation == "type_text" for c in kw["action_space"].candidates())
+ assert "laptop.pptx" in json.loads(kw["intent"])["next_action_hint"]
+ winner = (next(c for c in kw["action_space"].candidates() if c.id == "KEYPRESS_TAB")
+ if not selected else choice)
+ selected.append(winner)
+ return Ranking(winner, [], [], 0, 0, 1, True)
+
+ s.components.selector.rank = rank
+ first = _shell_observation(tmp_path / "first")
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(first)
+ assert s.act(first).action_type == "hotkey"
+ second = replace(_shell_observation(tmp_path / "second", step=1), last_receipt=gui_receipt())
+ assert s.act(second).parameters["text"] == "laptop.pptx"
+ s.components.grounder.assert_called_once()
+
+
+def test_scroll_invalidates_cached_coordinates_even_if_pixels_happen_to_match(tmp_path):
+ controller = GuiInteractionController()
+ obs = _shell_observation(tmp_path / "scene")
+ frame = {"state": ("same",), "screenshot": obs.screenshot.path}
+ assert controller.reusable_visual(frame, ("same",), Path(obs.screenshot.path))
+ controller.receive({"operation": "scroll"}, "unknown", False, 0)
+ assert not controller.reusable_visual(frame, ("same",), Path(obs.screenshot.path))
+
+
+@pytest.mark.parametrize("invalidating_id", ["SCROLL_DOWN", "GOAL_KEY:alt+tab", "GOAL_KEY:ctrl+o",
+ "GOAL_KEY:ctrl+l"])
+def test_new_grounding_after_invalidating_action_reaches_selector(tmp_path, invalidating_id):
+ s = session()
+ s.begin("Open the visible tool", requirements())
+ s.goal.memory.start(plan(), 0)
+ visual = ActionCandidate("visible-tool", "click", "Open the visible tool", source="visual_grounder",
+ metadata={"visual_point": [20, 40]})
+ s.components.grounder = Mock(return_value=[visual])
+ selected = []
+
+ def rank(**kw):
+ choices = kw["action_space"].candidates()
+ assert visual in choices
+ winner = next(c for c in choices if c.id == invalidating_id) if not selected else visual
+ selected.append(winner)
+ return Ranking(winner, [], [], 0, 0, 1, True)
+
+ s.components.selector.rank = rank
+ first = _shell_observation(tmp_path / "first")
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(first)
+ assert s.act(first).kind == "gui"
+ second = replace(_shell_observation(tmp_path / "second", step=1), last_receipt=gui_receipt())
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(second)
+ assert not s.goal.interaction.invalidate_visual
+ # Real workers refresh again without replaying the preceding scroll/hotkey.
+ refreshed = _shell_observation(tmp_path / "refreshed", step=1)
+ assert s.act(refreshed).parameters == {"x": 20, "y": 40}
+ assert selected == [selected[0], visual]
+ assert s.components.grounder.call_count == 2
+ assert not any(e.get("route") == "recovery_exhausted" for e in s.events)
+
+
+def test_field_identity_beats_spatial_hint_and_unrelated_nearby_text():
+ targets, _ = _space(FORM)
+ base = next(t for t in targets if t.name == "Title *")
+ fields = [replace(base, id="from", name="From", nearby_text="From", input_type="select-one"),
+ replace(base, id="to", name="To", nearby_text="Return date", input_type="select-one"),
+ replace(base, id="date", name="Departure date", nearby_text="To")]
+ values = [_input("Shanghai", "From city selector", label="Departure city"),
+ _input("Beijing", "To city selector", label="Arrival city"),
+ _input("March 25", "Departure date calendar immediately right of the To selector",
+ label="Departure date")]
+ assert [_input_target(v, fields).id for v in values] == ["from", "to", "date"]
+ space = goal_candidates(build_space(fields), fields, FORM, values)
+ assert {c.target_id: c.metadata["text"] for c in space.target_heads["type_text"]} == {
+ "from": "Shanghai", "to": "Beijing", "date": "March 25"}
+ frontier = input_frontier(space, fields, values)
+ assert not frontier["conflicting_fields"] and not frontier["unbound_inputs"]
+
+
+def test_duplicate_observed_labels_remain_unbound_instead_of_conflicting():
+ targets, _ = _space(FORM)
+ base = next(t for t in targets if t.name == "Title *")
+ fields = [replace(base, id="first"), replace(base, id="second")]
+ values = [_input("Known title", "Title")]
+ assert _input_target(values[0], fields) is None
+ space = goal_candidates(build_space(fields), fields, FORM, values)
+ frontier = input_frontier(space, fields, values)
+ assert not space.target_heads["type_text"] and not frontier["conflicting_fields"]
+ assert frontier["unbound_inputs"] == ["Title"]
+
+
+def test_small_input_change_is_visible_in_target_roi_but_not_global_average(tmp_path):
+ before, after = tmp_path / "before.png", tmp_path / "after.png"
+ image = Image.new("RGB", (1920, 1080), "white")
+ image.save(before)
+ ImageDraw.Draw(image).text((150, 1052), "Known command", fill="black")
+ image.save(after)
+ assert not visual_changed(before, after)
+ assert visual_changed(before, after, [130, 1048, 1600, 30])
+
+
+def test_visual_field_identity_does_not_change_with_detector_box_jitter():
+ a, b = _visual(_input()), _visual(_input(), (24, 46))
+ a.description, b.description = "Field at [10,40,30,10]", "Field at [14,41,30,10]"
+ assert _action_key(a) == _action_key(b)
+ b.metadata["text"] = "different.pptx"
+ assert _action_key(a) != _action_key(b)
+
+
+def test_command_has_separate_commit_and_no_premature_done():
+ controller = GuiInteractionController()
+ memory = TaskMemory("Compute", requirements())
+ value = _input('Example("known")', "Command input", kind="command", commit="enter")
+ stage = {**plan(), "id": "S1", "inputs": [value]}
+ targets, space = _space(FORM)
+ space.target_heads["visual"] = [_visual(value)]
+ controller.constrain(space, targets, stage, memory)
+ choice = space.target_heads["visual"][0]
+ key = choice.metadata["interaction_binding"]
+ controller.receive({"operation": "type_text", "interaction_binding": key}, "unknown", True, 0)
+ context = controller.constrain(space, targets, stage, memory)
+ commit = next(c for c in space.candidates() if c.metadata.get("commit_action"))
+ assert "ENTER" in context["next_intent"] and "done" not in space.controls
+ assert not any(c.operation == "type_text" for c in space.candidates())
+ action = gui_action(commit, None)
+ assert action.parameters == {"text": "\n", "input_mode": "insert", "click": [20, 45]}
+ controller.receive({"operation": "keypress", "interaction_binding": key,
+ "commit_action": True}, "unknown", True, 0)
+ assert controller.snapshot()["fields"][0]["phase"] == "submitted_unconfirmed"
+
+
+def test_native_select_uses_prefix_and_enter_instead_of_enumerating_countries():
+ raw, _ = _space(FORM)
+ field = replace(next(t for t in raw if t.name == "Title *"), name="Country of Citizenship",
+ role="combo_box", input_type="select-one", value="Afghanistan")
+ value = _input("China", "Country of Citizenship", kind="choice")
+ space = goal_candidates(build_space([field]), [field], FORM, [value])
+ choice = space.target_heads["type_text"][0]
+ action = gui_action(choice, field)
+ assert action.parameters["text"] == "China\n" and action.parameters["input_mode"] == "insert"
+ filled = goal_candidates(space, [replace(field, value="China")], FORM, [value])
+ assert not filled.target_heads["type_text"]
+
+
+def test_target_application_excludes_known_other_dock_apps_and_generalizes_to_geogebra():
+ assert application_mismatch("GeoGebra Classic 5",
+ '')
+ assert not application_mismatch("GeoGebra Classic 5",
+ '')
+ tree = '''
+
+
+
+ '''
+ targets, space = _space(tree)
+ GuiInteractionController().constrain(space, targets, {**plan(), "id": "S1", "inputs": [],
+ "target_application": "WPS Presentation"}, TaskMemory("Edit", requirements()))
+ offered = [c.description for c in space.candidates()]
+ assert any("Activities" in s for s in offered) and any("WPS 2019" in s for s in offered)
+ assert not any("LibreOffice" in s for s in offered)
+
+
+def test_original_error_survives_advice_and_blocks_same_method_across_work_items():
+ memory = TaskMemory("Import image", requirements())
+ memory.start(plan(), 0)
+ memory.apply(result(stage_outcome="not_met", unit_met=False, delivery_state="none",
+ summary="Image was not imported", remaining=["Try adding more arguments"], observed_errors=[{
+ "message": "Unknown command : Example", "command_name": "Example",
+ "application": "GeoGebra Classic 5", "evidence_refs": ["A2"]}]), {"A2": {}})
+ entry = next(f for f in memory.failures if f.get("observed_method"))
+ assert entry["error"] == "Unknown command : Example"
+ assert entry["suggested_actions"] == ["Try adding more arguments"]
+ value = {**plan(), "target_application": "GeoGebra Classic 5", "inputs": [_input('Example("other", 2)')]}
+ assert "unsupported command" in goal_error(value, memory, True)
+ memory.failures.extend({"error": "Another failure"} for _ in range(15))
+ assert planning_context(memory)["unsupported_methods"][0]["command_name"] == "Example"
+ assert not memory.blocked_command("OtherCommand(1)", "GeoGebra Classic 5")
+ assert not memory.blocked_command("Example(1)", "Another application")
+
+
+def test_navigation_never_becomes_content_delivery_or_resets_delivery_stall():
+ memory = TaskMemory("Edit", requirements())
+ memory.start(plan(), 0)
+ memory.apply(result(unit_met=False, progress_kind="navigation", delivery_state="implemented",
+ milestones=[{"key": "window-maximized", "summary": "Window maximized", "evidence_refs": ["A2"]}]),
+ {"A2": {}})
+ item = memory.work_items["R1:unit:1"]
+ assert item["delivery_state"] == "none" and not item["milestones"] and item["stalled_stages"] == 1
+
+
+def test_save_unit_can_finish_without_accepting_sibling_animation(tmp_path):
+ req = requirements()[0]
+ req.update(units=["File saved and open", "Animation matches tutorial"], unit_fact_keys=[[], []])
+ memory = TaskMemory("Save the file and build the animation", [req])
+ memory.start({**plan(), "purpose": "verify", "completion_criteria": "File saved and open"}, 0)
+ obs = make_observation(tmp_path, 2)
+
+ class Client:
+ config = SessionConfig()
+
+ def json_request(self, messages, validate, **kwargs):
+ context = json.loads(messages[1]["content"][0]["text"])
+ assert context["unit_acceptance"]["criterion"] == "File saved and open"
+ assert context["sibling_units_not_current_acceptance"][0]["goal"] == "Animation matches tutorial"
+ bad = _checked_result(unit_met=False)
+ bad["unit_assessment"]["status"] = "met"
+ assert not validate(bad)
+ good = _checked_result()
+ assert validate(good)
+ return good
+
+ checked = verify_goal(Client(), memory.instruction, Path(obs.screenshot.path), memory=memory,
+ history=[], end_reason="stage_done_requested", evidence={"A2": {}})
+ memory.apply(checked, {"A2": {}})
+ assert memory.work_items["R1:unit:1"]["status"] == "done"
+ assert memory.work_items["R1:unit:2"]["status"] == "pending"
+
+
+def test_input_uncertainty_does_not_skip_semantic_verification(tmp_path):
+ memory = TaskMemory("Submit a command", requirements())
+ memory.start(plan(), 0)
+ obs = make_observation(tmp_path, 2)
+ client = Mock(config=SessionConfig(), json_request=Mock(return_value=_checked_result(unit_met=False)))
+ verify_goal(client, "Submit a command", Path(obs.screenshot.path), memory=memory, evidence={"A2": {}},
+ history=[{"kind": "gui", "operation": "type_text", "relevant_change": False,
+ "screenshot": obs.screenshot.path}], end_reason="stage_done_requested")
+ client.json_request.assert_called_once()
+
+
+def test_last_interaction_is_preserved_as_historical_state_not_source_fact():
+ memory = TaskMemory("Edit", requirements())
+ memory.start(plan(), 0)
+ checkpoint = {"stage_id": "S1", "fields": [{"target": "Title", "phase": "typed_unconfirmed"}]}
+ memory.active_stage["interaction"] = checkpoint
+ memory.apply(result(unit_met=False, delivery_state="none"), {"A2": {}})
+ assert memory.snapshot()["last_interaction"] == checkpoint
+ assert planning_context(memory)["last_interaction_historical"] == checkpoint
+ assert not memory.facts
+
+
+def test_null_unit_assessment_is_a_contract_error_not_a_runtime_exception():
+ value = _checked_result()
+ value["unit_assessment"] = None
+ assert "unit_assessment" in result_error(value)
+
+
+def test_prepare_verifies_sources_without_model_delivery_assessment(tmp_path):
+ memory = TaskMemory("Read source material", requirements())
+ memory.start({**plan(), "purpose": "prepare"}, 0)
+ obs = make_observation(tmp_path, 2)
+
+ class Client:
+ config = SessionConfig()
+
+ def json_request(self, messages, validate, **kwargs):
+ context = json.loads(messages[1]["content"][0]["text"])
+ assert "unit_assessment" not in context["response_template"]
+ assert context["unit_acceptance"]["scope"] == "not_assessed_during_prepare"
+ source = result(unit_met=False, delivery_state="none", progress_kind="source", observed_errors=[])
+ assert validate(source)
+ # A model's redundant not_met delivery assessment must not cause another API call.
+ source["unit_assessment"] = {"work_item_id": "R1:unit:1", "status": "not_met",
+ "evidence_refs": [], "remaining": []}
+ assert validate(source)
+ assert not validate({**source, "unit_met": True, "delivery_state": "verified"})
+ assert not validate({**source, "facts": [{"key": "number", "value": 25}]})
+ return source
+
+ checked = verify_goal(Client(), memory.instruction, Path(obs.screenshot.path), memory=memory,
+ history=[], end_reason="stage_done_requested", evidence={"A2": {}})
+ assert checked["unit_assessment"] == {"work_item_id": "R1:unit:1", "status": "not_assessed",
+ "evidence_refs": [], "remaining": []}
+ assert checked["unit_met"] is False and checked["delivery_state"] == "none"
diff --git a/tests/harness/test_goal_loop_closure.py b/tests/harness/test_goal_loop_closure.py
new file mode 100644
index 0000000..ed8c388
--- /dev/null
+++ b/tests/harness/test_goal_loop_closure.py
@@ -0,0 +1,276 @@
+"""Task-independent regressions distilled from real GUI failures; not benchmark acceptance."""
+
+import json
+from dataclasses import replace
+from unittest.mock import Mock
+
+import pytest
+
+from tandem.gui.goal_candidates import _input_target, goal_candidates
+from tandem.gui.interaction import GuiInteractionController
+from tandem.gui.semantics import enrich_targets
+from tandem.memory.state import TaskMemory
+
+from .conftest import build_space
+from .test_goal_execution import _space
+from .test_goal_progress import FORM
+from .test_goal_stage import plan, requirements, result
+
+
+def _field():
+ targets, _ = _space(FORM)
+ return next(t for t in targets if t.name == "Title *")
+
+
+def test_specific_hint_does_not_bind_a_context_word_to_an_unrelated_select():
+ field = replace(_field(), name="Semester", role="combo_box", input_type="select-one")
+ value = {"label": "Semester 2 subclass with afternoon meeting", "target_hint": "Subclass B",
+ "text": "Subclass B", "input_mode": "replace", "kind": "choice"}
+ assert _input_target(value, [field]) is None
+
+
+def test_full_iso_date_never_binds_to_one_date_segment():
+ field = replace(_field(), name="Day", role="spin_button", description="dd")
+ value = {"label": "Visit date", "target_hint": "Visit date, focused Day segment",
+ "text": "2026-03-25", "input_mode": "replace"}
+ assert _input_target(value, [field]) is None
+
+
+def test_named_clickable_choice_is_not_translated_into_dropdown_typing():
+ field = replace(_field(), id="card", name="Subclass B (Class 123)", role="push_button")
+ value = {"label": "Afternoon subclass", "target_hint": "Subclass B", "text": "Subclass B",
+ "kind": "choice", "input_mode": "replace"}
+ space = goal_candidates(build_space([field]), [field], FORM, [value])
+ assert any(c.operation == "click" and c.target_id == "card" and c.source == "goal_input"
+ for c in space.candidates())
+ assert not space.target_heads["type_text"]
+
+
+@pytest.mark.parametrize("suffix", ["", " Visit date"])
+def test_a11y_date_segments_form_one_typed_input_without_cdp(suffix):
+ base = _field()
+ fields = [replace(base, id=f"part-{i}", name=name + suffix, role="spin_button", value=value,
+ x=100 + i * 40, y=200, width=30, height=20, nearby_text="Visit date")
+ for i, (name, value) in enumerate([("Day", "25"), ("Month", "3"), ("Year", "2026")])]
+ targets = enrich_targets(fields, None, (800, 600))
+ date = next(t for t in targets if t.input_type == "date")
+ assert date.name == "Visit date" and date.value == "2026-03-25"
+ assert date.date_order == ("day", "month", "year")
+
+
+def _controller():
+ memory = TaskMemory("Edit the current form", requirements())
+ memory.start(plan(), 0)
+ targets, _ = _space(FORM)
+ controller = GuiInteractionController()
+ controller.finalize(build_space(targets), targets, memory.active_stage, memory, FORM)
+ return controller, memory, targets
+
+
+def test_ui_change_does_not_erase_a_failed_path():
+ controller, _, targets = _controller()
+ controller.control.update(blocked_families=["activation_keys"], recovery_required=True)
+ pending = {"operation": "scroll", "action": {"parameters": {}},
+ "control_scope": controller.control["scope_key"], "control_state": controller.content_state}
+ controller.receive(pending, "unknown", True, 0,
+ targets=[replace(targets[0], name="Another visible section")])
+ assert controller.control["blocked_families"] == ["activation_keys"]
+ assert controller.control["recovery_required"]
+
+
+def test_changed_but_incorrect_input_is_not_treated_as_new_content_progress():
+ controller, _, targets = _controller()
+ controller.control.update(ground_calls=2)
+ pending = {"operation": "type_text", "action": {"parameters": {}},
+ "control_scope": controller.control["scope_key"], "control_state": controller.content_state}
+ controller.receive(pending, "not_met", True, 0,
+ targets=[replace(targets[0], value="Another wrong value")])
+ assert controller.local_effect != "controls_changed"
+ assert controller.control["without_effect"] == 1 and controller.control["ground_calls"] == 2
+
+
+def test_returning_to_seen_controls_is_not_endless_new_content():
+ controller, memory, targets = _controller()
+ base = targets[0]
+ for index in range(8):
+ pending = {"operation": "scroll", "action": {"parameters": {}},
+ "control_scope": controller.control["scope_key"],
+ "control_state": controller.content_state}
+ observed = [replace(base, name="Menu open" if index % 2 else "Menu closed")]
+ controller.receive(pending, "unknown", True, 0, targets=observed)
+ controller.finalize(build_space(observed), observed, memory.active_stage, memory, FORM)
+ assert controller.checkpoint_due
+
+
+def test_visual_dialog_name_variants_share_failure_and_grounding_budget():
+ from tandem.gui.candidates import ActionCandidate
+
+ controller, memory, _ = _controller()
+ tree = ''
+ controls = []
+ for name in ["EditorApp Replace dialog", "Replace", "Replace dialog"]:
+ space = goal_candidates(build_space([]), [], tree, [])
+ space.target_heads["visual"] = [ActionCandidate("close", "click", "Close", source="visual_grounder",
+ metadata={"visual_point": [30, 40], "blocking_overlay": True,
+ "overlay_name": name, "overlay_bounds": [10, 20, 100, 100]})]
+ controller.finalize(space, [], memory.active_stage, memory, tree)
+ controls.append(controller.control)
+ controller.control.update(blocked_families=["activation_keys"], ground_calls=2)
+ assert controls[0] is controls[1] is controls[2]
+ assert controls[-1]["ground_calls"] == 2
+
+
+def test_registered_source_key_does_not_promote_temporary_ui_state():
+ req = requirements()[0]
+ req.update(facts_needed=[{"key": "profile", "description": "Source profile"}],
+ unit_fact_keys=[["profile"]])
+ memory = TaskMemory("Fill a form from the profile", [req])
+ memory.start(plan(), 0)
+ update = result(stage_outcome="not_met", unit_met=False, delivery_state="none", facts=[{
+ "key": "profile", "value": "The current input is still empty", "status": "confirmed",
+ "scope": "stage", "source_ref": "Current form", "evidence_refs": ["A2"]}])
+ assert not memory.apply(update, {"A2": {}})
+ assert "profile" not in memory.facts
+ assert memory.last_rejections[0]["reason"] == "source_key_requires_source_fact"
+
+
+def test_task_scope_alone_does_not_make_a_form_snapshot_source_acquisition():
+ req = requirements()[0]
+ req.update(facts_needed=[{"key": "profile", "description": "Source profile"}],
+ unit_fact_keys=[["profile"]])
+ memory = TaskMemory("Fill a form", [req])
+ memory.start(plan(), 0)
+ update = result(stage_outcome="not_met", unit_met=False, delivery_state="none", facts=[{
+ "key": "profile", "value": "Current form is still wrong", "status": "confirmed",
+ "scope": "task", "source_ref": "Form", "evidence_refs": ["A2"]}])
+ assert not memory.apply(update, {"A2": {}})
+ assert "profile" not in memory.facts
+ assert memory.last_rejections[0]["reason"] == "source_update_requires_acquisition_stage"
+
+
+def test_business_dialog_preserves_its_local_objective():
+ controller, memory, targets = _controller()
+ memory.active_stage["goal"] = "Select the requested afternoon class"
+ memory.active_stage["completion_criteria"] = "Select the requested afternoon class"
+ modal = (''
+ '')
+ space = goal_candidates(build_space(targets), targets, modal, [])
+ context = controller.finalize(space, targets, memory.active_stage, memory, modal)
+ assert "dismiss" not in context["next_intent"].lower()
+ assert "afternoon class" in context["next_intent"]
+
+
+def test_selector_recovery_context_does_not_keep_the_original_navigation_method():
+ controller, memory, _ = _controller()
+ context = {"goal": "Edit a chart", "completion_criteria": "Chart saved", "inputs": [],
+ "method": "Switch applications and edit the chart", "guidance": "Read another page",
+ "next_intent": "Close the blocking confirmation", "blocking_overlay": "Confirmation",
+ "scope_recovery": True, "constraints": [], "unbound_inputs": [], "execution_control": {}}
+ compact = controller.choice_context(context)
+ assert compact["objective"] == "Close the blocking confirmation"
+ assert not compact["method"] and not compact["guidance"]
+ assert compact["delegated_goal"] == context["goal"]
+ assert "Switch applications" not in json.dumps(compact)
+
+
+def test_unresolved_scope_recovery_omits_future_edit_values_and_exit_conditions():
+ controller, _, _ = _controller()
+ context = {"goal": "Edit chart", "completion_criteria": "Chart has future metrics and is saved",
+ "inputs": [{"label": "Later title", "text": "Next quarter", "input_mode": "replace"}],
+ "next_intent": "Restore input access by resolving the current dialog",
+ "blocking_overlay": "Confirmation", "scope_recovery": True, "unbound_inputs": ["Later title"]}
+ compact = controller.choice_context(context)
+ assert compact["inputs"] == [] and compact["unbound_inputs"] == []
+ assert "future metrics" not in json.dumps(compact) and "Next quarter" not in json.dumps(compact)
+
+
+def test_verify_navigation_cannot_reclaim_a_historical_save(tmp_path):
+ from PIL import Image
+
+ from tandem.verification.goal_verifier import verify_goal
+
+ _, memory, _ = _controller()
+ path = tmp_path / "screen.png"
+ Image.new("RGB", (16, 16)).save(path)
+ response = result(stage_outcome="not_met", unit_met=False, delivery_state="saved",
+ progress_kind="navigation", observed_errors=[], milestones=[])
+ response["unit_assessment"] = {"work_item_id": "R1:unit:1", "status": "not_met",
+ "remaining": [], "evidence_refs": ["A2"]}
+ client = Mock()
+ client.config.stage_verify_max_chars = 48000
+
+ def request(messages, validate, **kw):
+ assert validate(response)
+ return response
+
+ client.json_request.side_effect = request
+ output = verify_goal(client, "Edit", path, memory=memory, history=[],
+ end_reason="stage_done_requested", evidence={"A2": {"screenshot": str(path)}})
+ assert output["delivery_state"] == "none" and not output["unit_met"]
+ assert output["historical_delivery_claim"] == "saved"
+
+
+def test_invalid_verify_output_is_uncertain_not_task_success_or_api_failure(tmp_path):
+ from PIL import Image
+
+ from tandem.models.slow import ModelResponseError
+ from tandem.verification.goal_verifier import verify_goal
+
+ _, memory, _ = _controller()
+ path = tmp_path / "screen.png"
+ Image.new("RGB", (16, 16)).save(path)
+ client = Mock()
+ client.config.stage_verify_max_chars = 48000
+ client.json_request.side_effect = ModelResponseError("invalid structured response")
+ output = verify_goal(client, "Edit", path, memory=memory, history=[],
+ end_reason="stage_done_requested", evidence={"A2": {"screenshot": str(path)}})
+ assert output["stage_outcome"] == "uncertain" and not output["unit_met"]
+ assert output["failure_kind"] == "verification_output_invalid"
+ assert not memory.apply(output, {"A2": {}})
+
+
+def test_missing_current_target_triggers_grounding_despite_other_controls(tmp_path):
+ import pytest
+
+ from tandem.config import SessionConfig
+ from tandem.gui.candidates import ActionCandidate
+ from tandem.models.ranking import Ranking
+ from tandem.runtime.goal_stage import ObservationRequired
+ from tandem.runtime.session import TandemSession
+
+ from .conftest import make_components, make_observation
+
+ components = make_components(lambda *a, **kw: plan(), lambda *a, **kw: result(), Mock())
+ components.memory_factory = TaskMemory
+ components.grounder = Mock(return_value=[ActionCandidate(
+ "located", "click", "Visible requested option", source="visual_grounder",
+ metadata={"visual_point": [20, 40]})])
+ components.selector.rank = Mock(return_value=Ranking(
+ components.grounder.return_value[0], [], [], 0, 0, 1, True))
+ session = TandemSession(components, config=SessionConfig(execution_mode="goal"))
+ session.begin("Choose a visible option", requirements())
+ session.goal.memory.start({**plan(), "inputs": [{"label": "Requested option", "text": "Option B",
+ "target_hint": "Requested option", "kind": "choice", "input_mode": "replace"}]}, 0)
+ observation = make_observation(tmp_path, 0)
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ session.act(observation)
+ assert session.act(observation).parameters == {"x": 20, "y": 40}
+ components.grounder.assert_called_once()
+
+
+def test_invalid_planner_output_is_an_agent_stop_not_an_external_api_failure(tmp_path):
+ from tandem.config import SessionConfig
+ from tandem.models.slow import ModelResponseError
+ from tandem.runtime.session import TandemSession
+
+ from .conftest import make_components, make_observation
+
+ planner = Mock(side_effect=ModelResponseError("Unrepairable structured output"))
+ components = make_components(planner, Mock(), Mock())
+ components.memory_factory = TaskMemory
+ session = TandemSession(components, config=SessionConfig(execution_mode="goal"))
+ session.begin("Edit", requirements())
+ action = session.act(make_observation(tmp_path, 0))
+ assert action.kind == "fail" and action.reason == "planner_output_invalid"
+ assert session.events[-1]["route"] == "planner_output_invalid"
diff --git a/tests/harness/test_goal_mechanisms.py b/tests/harness/test_goal_mechanisms.py
new file mode 100644
index 0000000..1268c37
--- /dev/null
+++ b/tests/harness/test_goal_mechanisms.py
@@ -0,0 +1,355 @@
+"""Synthetic mechanism regressions, not real task acceptance or model quality measurements."""
+
+import json
+from dataclasses import replace
+from pathlib import Path
+from unittest.mock import Mock
+
+import pytest
+from PIL import Image
+
+from tandem.gui.candidates import ActionCandidate
+from tandem.gui.compile import gui_action
+from tandem.gui.goal_candidates import application_mismatch, document_frontier, input_frontier, modal_frontier
+from tandem.gui.grounding import ground_visual
+from tandem.gui.semantics import enrich_targets
+from tandem.memory.context import planning_context
+from tandem.memory.state import TaskMemory
+from tandem.models.ranking import Ranking
+from tandem.planning.goal_planner import goal_error, goal_requirements, plan_goal
+from tandem.runtime.goal_stage import _action_key, _target_effect, _visual_changed
+from tandem.verification.goal_verifier import result_error, verify_goal
+
+from .conftest import build_space, make_observation
+from .test_goal_execution import _space
+from .test_goal_memory import session
+from .test_goal_progress import FORM
+from .test_goal_stage import plan, requirements, result
+
+
+def test_field_alias_is_unique_and_already_filled_value_is_recognized():
+ tree = FORM.replace("Title *", "Brief diagnosis summary (required)").replace(
+ 'size="(200,30)"/>', 'size="(200,30)">Known diagnosis', 1)
+ inputs = [{"label": "Diagnosis", "target_hint": "Diagnosis Summary", "text": "Known diagnosis",
+ "input_mode": "replace"}]
+ targets, space = _space(tree, inputs)
+ frontier = input_frontier(space, targets, inputs)
+ assert frontier["satisfied_inputs"] == ["Diagnosis"]
+ assert not frontier["unbound_inputs"] and not space.target_heads["type_text"]
+ targets, space = _space(tree.replace("Issue number", "Diagnosis Summary"), inputs)
+ assert not space.target_heads["type_text"] # A broad label never resolves ambiguous fields.
+
+
+@pytest.mark.parametrize("order", [("month", "day", "year"), ("day", "month", "year")])
+@pytest.mark.parametrize("suffix", ["", " Departure date"])
+def test_native_date_uses_observed_segments_and_checks_iso_value(order, suffix):
+ targets, space = _space(FORM)
+ original = next(t for t in targets if t.name == "Title *")
+ snapshot = {"elements": [{"index": 1, "role": "entry", "name": "Departure date",
+ "input_type": "date", "date_order": order, "value": "", "label_source": "dom",
+ "x": original.x, "y": original.y, "width": original.width, "height": original.height,
+ "operations": ["click", "type_text"]}]}
+ targets = [replace(t, application="Google Chrome") for t in targets]
+ target = next(t for t in enrich_targets(targets, FORM, (1920, 1080), snapshot)
+ if t.name == "Departure date")
+ from tandem.gui.goal_candidates import goal_candidates
+
+ inputs = [{"label": "Travel date", "target_hint": "Departure date", "text": "2026-03-25",
+ "input_mode": "replace"}]
+ parts = [replace(target, id=f"part-{name}", role="spin_button", name=name.title() + suffix, input_type="",
+ x=target.x + index * 40, width=30, value="0") for index, name in enumerate(order)]
+ for name, expected in [("Year", "2026"), ("Month", "03"), ("Day", "25")]:
+ space = goal_candidates(build_space([target, *parts]), [target, *parts], FORM, inputs)
+ choice = space.target_heads["type_text"][0]
+ segment = next(p for p in parts if p.id == choice.target_id)
+ action = gui_action(choice, segment)
+ assert segment.name == name + suffix and action.parameters["text"] == expected
+ assert action.parameters["click"][0] == segment.x + segment.width // 2
+ parts = [replace(p, value=expected) if p.id == segment.id else p for p in parts]
+ assert not goal_candidates(space, [replace(target, value="2026-03-25")], FORM,
+ inputs).target_heads["type_text"]
+ inputs[0]["text"] = "03252026"
+ assert "ISO" in input_frontier(space, [target], inputs)["conflicting_fields"][0]
+
+
+def test_portal_dialog_survives_stale_foreground_metadata():
+ tree = '''
+
+
+ '''
+ targets, space = _space(tree)
+ assert "Select" in [t.name for t in targets] and "Trash" not in [t.name for t in targets]
+ context = modal_frontier(space, targets, tree)
+ assert context["blocking_overlay"] == "Open File"
+ assert any(c.target_id for c in space.target_heads["click"])
+ assert not space.target_heads["scroll"] and "done" not in space.controls
+
+
+def test_visual_modal_keeps_close_but_removes_background_and_false_progress(tmp_path):
+ targets, space = _space(FORM)
+ close = ActionCandidate(id="close-overlay", operation="click", description="Close the modal",
+ source="visual_grounder", metadata={"visual_point": [350, 250],
+ "blocking_overlay": True, "overlay_name": "System Check",
+ "overlay_bounds": [200, 200, 200, 100]})
+ space.target_heads["visual"] = [close]
+ context = modal_frontier(space, targets, FORM)
+ assert space.target_heads["visual"] == [close] and not space.target_heads["scroll"]
+ assert "System Check" in context["next_intent"]
+ before, after = tmp_path / "before.png", tmp_path / "after.png"
+ Image.new("RGB", (600, 600), "black").save(before)
+ frame = Image.new("RGB", (600, 600), "white")
+ frame.paste("black", (200, 200, 400, 300))
+ frame.save(after)
+ assert _visual_changed(before, after)
+ assert not _visual_changed(before, after, context["overlay_bounds"])
+
+
+def test_grounder_records_explicit_overlay_without_inventing_one(tmp_path):
+ obs = make_observation(tmp_path, 0)
+ client = Mock()
+ client.json_request.return_value = {"blocking_overlay": {"name": "Warning", "bounds": [1, 1, 40, 40]},
+ "candidates": [{"operation": "click", "target": "Close", "bounds": [5, 5, 20, 20]}]}
+ choices = ground_visual(client, "Edit", {"action_proposals": []}, Path(obs.screenshot.path))
+ assert choices[0].metadata["overlay_name"] == "Warning"
+
+
+def test_wrong_document_can_navigate_but_cannot_receive_article_text():
+ inputs = [{"label": "Page", "target_hint": "Address and search bar",
+ "text": "file:///home/user/right.html", "input_mode": "replace"},
+ {"label": "Title", "target_hint": "Title", "text": "Article title", "input_mode": "replace"}]
+ targets, space = _space(FORM, inputs)
+ context = document_frontier(space, targets, "right.html")
+ assert context["document_recovery"] == "right.html"
+ assert [c.metadata["text"] for c in space.target_heads["type_text"]] == [inputs[0]["text"]]
+ assert "done" not in space.controls
+
+
+def test_runtime_gives_jev_a_navigation_chance_on_wrong_document(tmp_path):
+ s = session()
+ s.begin("Fill right.html", requirements())
+ s.goal.memory.start({**plan(), "target_document": "right.html", "inputs": [
+ {"label": "Page", "target_hint": "Address and search bar",
+ "text": "file:///home/user/right.html", "input_mode": "replace"}]}, 0)
+
+ def rank(**kwargs):
+ assert "right.html" in kwargs["stage_goal"] and "Navigate" in kwargs["stage_goal"]
+ winner = kwargs["action_space"].target_heads["type_text"][0]
+ return Ranking(winner, [], [], 0, 0, 1, True)
+
+ s.components.selector.rank = rank
+ action = s.act(make_observation(tmp_path, 0, tree_xml=FORM))
+ assert action.parameters["text"] == "file:///home/user/right.html"
+
+
+def test_date_effect_checks_intended_field_even_after_tab_moves_focus():
+ targets, _ = _space(FORM)
+ target = next(t for t in targets if t.name == "Title *")
+ pending = {"operation": "type_text", "target_semantic": target.semantic_key,
+ "field_label": target.name, "expected_text": "2026-03-25", "input_mode": "replace"}
+ targets = [replace(t, value="2026-03-25") if t.id == target.id else t for t in targets]
+ assert _target_effect(pending, targets) == "met"
+ assert _target_effect(pending, [replace(t, value="") for t in targets]) == "not_met"
+
+
+def test_partial_milestones_advance_once_without_verifying_entire_unit():
+ memory = TaskMemory("Fill application", requirements())
+ for key in ["application/section-1", "application/section-2", "application/section-2"]:
+ memory.start(plan(), 0)
+ advanced = memory.apply(result(unit_met=False, milestones=[{
+ "key": key, "summary": "Section saved and checked", "evidence_refs": ["A2"]}]), {"A2": {}})
+ item = memory.work_items["R1:unit:1"]
+ assert not advanced and item["stalled_stages"] == 1
+ assert len(item["milestones"]) == 2 and memory.stats["milestone_advances"] == 2
+ assert item["delivery_state"] != "verified" and item["status"] != "done"
+ assert planning_context(memory)["ready_work_items"][0]["milestones"] == item["milestones"]
+ memory.start(plan(), 0)
+ memory.apply(result(stage_outcome="uncertain", unit_met=False, delivery_state="none", milestones=[{
+ "key": "unsupported", "summary": "Not observed", "evidence_refs": ["A2"]}]), {"A2": {}})
+ assert "unsupported" not in item["milestones"]
+
+
+def test_failed_cli_writes_require_inspection_before_another_write():
+ memory = TaskMemory("Edit deck", requirements())
+ for _ in range(2):
+ memory.start({**plan(), "route": "cli", "command": "different failed command"}, 0)
+ memory.apply(result(stage_outcome="uncertain", unit_met=False, delivery_state="none",
+ failure_kind="execution_error"), {"A2": {}})
+ candidate = {**plan(), "route": "cli", "command": "another guessed edit"}
+ assert "two CLI" in goal_error(candidate, memory, True)
+ assert not goal_error({**candidate, "purpose": "prepare"}, memory, True)
+
+
+def test_task_gui_restriction_overrides_cli_enabled_configuration(tmp_path):
+ s = session()
+ s.begin("Book tickets", requirements())
+
+ class Client:
+ def json_request(self, messages, validate, **kwargs):
+ context = json.loads(messages[1]["content"][0]["text"])
+ assert context["allow_cli"] is False
+ assert not validate({**plan(), "route": "cli", "command": "cat source"})
+ assert validate(plan())
+ return plan()
+
+ obs = make_observation(tmp_path, 0)
+ plan_goal(Client(), "Do not open or use Terminal, VSCode or backend APIs.",
+ Path(obs.screenshot.path), [], [], "", memory=s.goal.memory)
+
+
+@pytest.mark.parametrize("violation", ["", "Edited website source to bypass field validation"])
+def test_cli_integrity_is_not_gui_acceptance_and_policy_violations_are_rejected(tmp_path, violation):
+ s = session()
+ s.begin("Edit presentation", requirements())
+ s.goal.memory.start({**plan(), "route": "cli", "purpose": "save"}, 0)
+ obs = make_observation(tmp_path, 1)
+ client = Mock()
+ client.config.stage_verify_max_chars = 12000
+ client.json_request.return_value = result(policy_violation=violation,
+ policy_constraint="website_source_tampering", policy_evidence_refs=["A2"])
+ checked = verify_goal(client, "Edit", Path(obs.screenshot.path), memory=s.goal.memory,
+ evidence={"A2": {}},
+ history=[{"kind": "cli", "evidence_ref": "A2", "screenshot": obs.screenshot.path}],
+ end_reason="cli_returned")
+ assert checked["unit_met"] is False
+ assert checked["delivery_state"] == ("none" if violation else "saved")
+ if violation:
+ assert checked["failure_kind"] == "policy_violation" and checked["milestones"] == []
+ assert result_error(result(milestones=[{"key": "x", "summary": "saved", "evidence_refs": [{}]}]))
+
+
+def test_policy_violation_ends_attempt_instead_of_replanning_around_the_rule(tmp_path):
+ s = session()
+ s.begin("Fill website", requirements())
+ s.goal.memory.start(plan(), 0)
+ s.goal.end_reason = "returned_to_planner"
+ s.components.stage_verifier = Mock(return_value=result(
+ stage_outcome="not_met", unit_met=False, delivery_state="none", evidence_refs=["S1:O0"],
+ failure_kind="policy_violation", policy_violation="Changed application validation",
+ remaining=["Changed application validation"]))
+ action = s.act(make_observation(tmp_path, 0))
+ assert action.kind == "fail" and "policy_violation" in action.reason
+
+
+def test_duplicate_requirements_are_repaired_without_discarding_either_delivery():
+ first = {**requirements()[0], "requirement": "Create report", "units": ["Report saved"]}
+ second = {**first, "requirement": "Create slides", "units": ["Slides saved"]}
+
+ class Client:
+ def json_request(self, messages, validate, **kwargs):
+ assert not validate({"requirements": [first, second]})
+ assert "Duplicate" in messages[-1]["content"]
+ repaired = {"requirements": [first, {**second, "id": "R2"}]}
+ assert validate(repaired)
+ return repaired
+
+ with pytest.raises(ValueError, match="unique"):
+ TaskMemory("Create report and slides", [first, second])
+ memory = TaskMemory("Create report and slides", goal_requirements(Client(), "Create report and slides"))
+ assert [item["goal"] for item in memory.work_items.values()] == ["Report saved", "Slides saved"]
+
+
+def test_closed_portal_cannot_take_over_chrome_or_remove_its_controls():
+ tree = '''
+
+ '''
+ targets, space = _space(tree)
+ assert [t.name for t in targets] == ["Save"]
+ assert modal_frontier(space, targets, tree) == {}
+ assert any(c.target_id for c in space.target_heads["click"])
+ assert "KEYPRESS_ESCAPE" in [c.id for c in space.target_heads["keypress"]]
+
+
+def test_anonymous_shell_strip_is_not_renamed_to_activities():
+ tree = '''
+
+
+ '''
+ from tandem.gui.targets import A11yTargetBuilder
+
+ raw = A11yTargetBuilder().build(tree, "t", 128, infer_button_labels=True)
+ targets = enrich_targets(raw, tree, (1920, 1080))
+ assert not any(t.name == "Activities" and t.role == "push_button" for t in targets)
+
+
+def test_wrong_application_preserves_actual_navigation_shortcuts_not_editing():
+ assert application_mismatch("WPS Presentation",
+ '')
+ assert not application_mismatch("WPS Presentation", '')
+ targets, space = _space(FORM)
+ document_frontier(space, targets, application="WPS Presentation")
+ keys = {c.id for c in space.target_heads["keypress"]}
+ assert {"GOAL_KEY:alt+tab", "GOAL_KEY:ctrl+o", "KEYPRESS_ESCAPE"} <= keys
+ assert "GOAL_KEY:ctrl+s" not in keys and not space.target_heads["type_text"]
+ launcher_tree = '''
+
+ '''
+ targets, space = _space(launcher_tree)
+ document_frontier(space, targets, application="WPS Presentation")
+ assert any("Activities" in c.description for c in space.target_heads["click"])
+
+
+def test_visual_date_cannot_bypass_observed_native_date_adapter(tmp_path):
+ client = Mock()
+ proposal = {"operation": "type_text", "target_hint": "Incident Time", "text": "2024-04-25",
+ "input_mode": "replace"}
+ client.json_request.return_value = {"candidates": [{**proposal, "target": "Incident Time",
+ "bounds": [1, 1, 20, 20]}]}
+ obs = make_observation(tmp_path, 0)
+ assert not ground_visual(client, "Fill date", {"action_proposals": [proposal]},
+ Path(obs.screenshot.path))
+
+
+@pytest.mark.parametrize("constraint,refs", [("", ["A2"]), ("Use keyboard navigation", ["A2"]),
+ ("website_source_tampering", ["old"])])
+def test_planner_method_or_unproven_policy_error_is_recoverable(tmp_path, constraint, refs):
+ s = session()
+ s.begin("Fill form", requirements())
+ s.goal.memory.start(plan(), 0)
+ obs = make_observation(tmp_path, 1)
+ client = Mock()
+ client.config.stage_verify_max_chars = 12000
+ client.json_request.return_value = result(policy_violation="Used a wrong shortcut",
+ policy_constraint=constraint, policy_evidence_refs=refs)
+ checked = verify_goal(client, "Fill form", Path(obs.screenshot.path), memory=s.goal.memory,
+ evidence={"A2": {}}, history=[{"kind": "gui", "evidence_ref": "A2",
+ "observed_change": True, "screenshot": obs.screenshot.path}],
+ end_reason="stage_done_requested")
+ assert checked["failure_kind"] == "recoverable_stage_error" and not checked["unit_met"]
+
+
+def test_duplicate_buttons_keep_row_context_and_identity_after_layout_shift():
+ from tandem.gui.candidates import ActionCandidateBuilder
+
+ snapshot = {"elements": [{"index": index, "role": "push_button", "name": "Select",
+ "nearby_text": row, "x": 100, "y": 100 * index, "width": 80, "height": 30,
+ "operations": ["click"]} for index, row in enumerate(["G8 09:00 North", "G14 11:00 South"], 1)]}
+ targets = enrich_targets([], None, (1920, 1080), snapshot)
+ space = ActionCandidateBuilder().build(targets, context="G14")
+ choices = space.target_heads["click"]
+ assert "G14" in choices[0].description and "G8" in choices[1].description
+ assert _action_key(choices[0], targets) != _action_key(choices[1], targets)
+ moved = [replace(t, y=t.y + 100) for t in targets]
+ assert _action_key(choices[0], moved) == _action_key(choices[0], targets)
+
+
+def test_working_artifact_cannot_be_silently_replaced_by_a_fresh_copy():
+ memory = TaskMemory("Edit presentation", requirements())
+ artifact = {"target_path": "/output/deck.pptx", "working_path": "/work/current.pptx",
+ "save_state": "saved", "evidence_refs": ["A2"]}
+ for change in [artifact, {**artifact, "working_path": "/work/new.pptx"}]:
+ memory.start(plan(), 0)
+ memory.apply(result(unit_met=False, delivery_state="saved", artifacts=[change]), {"A2": {}})
+ assert memory.artifacts[artifact["target_path"]]["working_path"] == artifact["working_path"]
+ assert memory.last_rejections[-1]["reason"] == "working_copy_change_requires_explicit_migration"
+ memory.start(plan(), 0)
+ memory.apply(result(unit_met=False, delivery_state="saved", artifacts=[{
+ **artifact, "working_path": "/work/new.pptx", "replaces_working_path": artifact["working_path"]}]),
+ {"A2": {}})
+ assert memory.artifacts[artifact["target_path"]]["working_path"] == "/work/new.pptx"
diff --git a/tests/harness/test_goal_memory.py b/tests/harness/test_goal_memory.py
new file mode 100644
index 0000000..195837d
--- /dev/null
+++ b/tests/harness/test_goal_memory.py
@@ -0,0 +1,275 @@
+"""Synthetic task-memory and local recovery regressions, not benchmark results."""
+
+from unittest.mock import Mock, patch
+
+import pytest
+from PIL import Image
+
+from tandem.config import SessionConfig
+from tandem.gui.candidates import ActionCandidate, ActionCandidateBuilder
+from tandem.gui.goal_candidates import foreground_tree, goal_candidates, scroll_point
+from tandem.gui.targets import A11yTargetBuilder
+from tandem.memory.context import planning_context
+from tandem.memory.state import TaskMemory
+from tandem.models.ranking import Ranking
+from tandem.protocol.models import GuiAction
+from tandem.runtime.goal_stage import ObservationRequired
+from tandem.runtime.session import TandemSession
+from tandem_eval.osworld import gui_code
+
+from .conftest import FixedSelector, gui_receipt, make_components, make_observation
+from .test_goal_stage import plan, requirements, result
+
+
+def fact(key="quota", value="1400000 USD", **fields):
+ return {"key": key, "value": value, "status": "confirmed", "scope": "task",
+ "source_ref": "source.pdf page 1", "evidence_refs": ["A2"], **fields}
+
+
+def source_memory():
+ req = requirements()[0]
+ req.update(facts_needed=[{"key": "quota", "description": "Annual quota"}],
+ unit_fact_keys=[["quota"]])
+ return TaskMemory("Edit", [req])
+
+
+def prepare(memory, update):
+ memory.start({**plan(), "work_item_id": "R1:prepare", "purpose": "prepare"}, 0)
+ memory.apply(result(stage_outcome="not_met", unit_met=False, delivery_state="none",
+ facts=[update]), {"A2": {}})
+
+
+def test_registered_source_is_reusable_when_marked_as_source_content():
+ memory = source_memory()
+ prepare(memory, fact())
+ assert memory.facts["quota"]["scope"] == "task"
+ assert memory.work_items["R1:prepare"]["status"] == "done"
+ assert memory.ready_items()[0]["id"] == "R1:unit:1"
+
+
+def test_uncertain_observation_and_unexplained_change_preserve_source():
+ memory = source_memory()
+ prepare(memory, fact())
+ for update in [fact(value="Not currently visible", status="uncertain"), fact(value="Guess")]:
+ memory.start({**plan(), "purpose": "prepare"}, 1)
+ memory.apply(result(unit_met=False, delivery_state="none", facts=[update]), {"A2": {}})
+ assert memory.facts["quota"]["value"] == "1400000 USD"
+ assert memory.last_rejections[0]["reason"] == "confirmed_source_preserved"
+ memory.start({**plan(), "purpose": "prepare"}, 2)
+ memory.apply(result(unit_met=False, delivery_state="none",
+ facts=[fact(value="1500000 USD", correction_reason="Source correction observed")]),
+ {"A2": {}})
+ assert memory.facts["quota"]["value"] == "1500000 USD"
+
+
+def test_independent_unit_unlocks_before_all_source_reading_finishes():
+ req = requirements()[0]
+ req.update(units=["Save quota page", "Save sales page"], unit_fact_keys=[["quota"], ["sales"]],
+ facts_needed=[{"key": "quota", "description": "Quota"},
+ {"key": "sales", "description": "Sales"}])
+ memory = TaskMemory("Edit", [req])
+ prepare(memory, fact())
+ ready = {w["id"] for w in memory.ready_items()}
+ assert "R1:unit:1" in ready and "R1:unit:2" in ready
+ assert not memory._facts_ready(memory.work_items["R1:unit:2"])
+ assert memory.work_items["R1:prepare"]["status"] != "done"
+
+
+def test_material_only_update_does_not_reset_delivery_stall():
+ memory = TaskMemory("Edit", requirements())
+ memory.start(plan(), 0)
+ memory.apply(result(stage_outcome="not_met", unit_met=False, delivery_state="none",
+ materials=[{"source_ref": "frame.png", "covered_parts": ["another screenshot"],
+ "summary": "No new required facts", "evidence_refs": ["A2"]}]),
+ {"A2": {}})
+ assert memory.stats["source_parts"] == 1
+ assert memory.stats["stages_without_progress"] == 1
+
+
+def test_observed_missing_module_is_not_lost_when_recent_failure_window_rotates():
+ memory = TaskMemory("Edit", requirements())
+ memory.failures = [{"error": "ModuleNotFoundError: No module named 'pptx'"}]
+ memory.failures.extend({"error": "A later GUI failure"} for _ in range(15))
+ context = planning_context(memory)
+ assert context["observed_missing_modules"] == ["pptx"]
+ assert all("pptx" not in f["error"] for f in context["failures"])
+
+
+def test_unit_can_reuse_source_definition_from_another_requirement():
+ req = requirements()[0]
+ req.update(facts_needed=[{"key": "quota", "description": "Quota"}], unit_fact_keys=[["quota"]])
+ other = {**req, "id": "R2", "facts_needed": []}
+ memory = TaskMemory("Edit", [req, other])
+ assert memory.work_items["R2:unit:1"]["facts_needed"] == req["facts_needed"]
+ assert not memory._facts_ready(memory.work_items["R2:unit:1"])
+ prepare(memory, fact())
+ assert "R2:unit:1" in {w["id"] for w in memory.ready_items()}
+
+
+def test_constraint_defined_source_gets_an_executable_preparation_item():
+ delivery = requirements()[0]
+ delivery["unit_fact_keys"] = [["quota"]]
+ constraint = {**delivery, "id": "R2", "kind": "constraint", "units": [], "unit_fact_keys": [],
+ "facts_needed": [{"key": "quota", "description": "Quota from the source"}]}
+ memory = TaskMemory("Edit", [delivery, constraint])
+ assert {w["id"] for w in memory.ready_items()} == {"R1:prepare", "R1:unit:1"}
+ assert not memory._facts_ready(memory.work_items["R1:unit:1"])
+ assert memory.work_items["R1:prepare"]["facts_needed"] == constraint["facts_needed"]
+ prepare(memory, fact())
+ assert [w["id"] for w in memory.ready_items()] == ["R1:unit:1"]
+
+
+def test_source_can_cite_registered_history_but_completion_needs_current_evidence():
+ memory = source_memory()
+ prepare(memory, fact())
+ memory.start(plan(), 1)
+ memory.apply(result(evidence_refs=["A2"]), {"A3": {}})
+ assert memory.work_items["R1:unit:1"]["status"] != "done"
+ memory.start(plan(), 2)
+ memory.apply(result(evidence_refs=["A2", "A3"]), {"A3": {}})
+ assert memory.work_items["R1:unit:1"]["status"] == "done"
+
+
+def test_named_images_and_native_file_targets_survive_bounded_selection():
+ chrome = ''
+ chrome += ''.join(f''
+ for i in range(150))
+ chrome += ''
+ chrome += ''
+ targets = A11yTargetBuilder().build(chrome, "s", 8, context="Open Fractal tutorial", filter_occluded=True)
+ space = ActionCandidateBuilder().build(targets, context="Open Fractal tutorial")
+ assert any("Fractal tutorial" in c.description for c in space.target_heads["click"])
+ files = '''
+
+
+ '''
+ targets = A11yTargetBuilder().build(files, "s", filter_occluded=True)
+ assert {t.name for t in targets} == {"Desktop", "documents"}
+ space = goal_candidates(ActionCandidateBuilder().build(targets), targets, files, [])
+ assert space.target_heads["double_click"]
+
+
+def test_scroll_point_is_inside_the_small_foreground_window():
+ tree = ''
+ x, y = scroll_point(tree, (1920, 1080))
+ assert 94 < x < 746 and 54 < y < 706
+
+
+def test_x11_foreground_identity_excludes_stale_background_active_app():
+ tree = '''
+
+
+
+ '''
+ filtered = foreground_tree(tree)
+ assert 'name="Google Chrome"' not in filtered
+ assert 'name="wpp"' in filtered and 'name="gnome-shell"' in filtered
+
+
+def session():
+ components = make_components(lambda *a, **kw: plan(), lambda *a, **kw: None, FixedSelector())
+ components.memory_factory = TaskMemory
+ components.stage_verifier = lambda *a, **kw: result(unit_met=False, delivery_state="none")
+ return TandemSession(components, config=SessionConfig(execution_mode="goal"))
+
+
+def test_no_effect_forces_grounding_and_masks_the_failed_candidate(tmp_path):
+ s = session()
+ s.begin("Edit", requirements())
+ grounded = ActionCandidate("visual", "click", "Observed alternate", source="visual_grounder",
+ metadata={"visual_point": [500, 500]})
+ s.components.grounder = Mock(return_value=[grounded])
+ obs = make_observation(tmp_path, 0)
+ with pytest.raises(ObservationRequired):
+ s.act(obs)
+ s.act(obs)
+ s.act(make_observation(tmp_path, 1, receipt=gui_receipt()))
+ failed = s.goal.pending["candidate_key"]
+ obs2 = make_observation(tmp_path, 2, receipt=gui_receipt())
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(obs2)
+ s.components.grounder.assert_called_once()
+
+ def rank(**kwargs):
+ from tandem.runtime.goal_stage import _action_key
+
+ candidates = kwargs["action_space"].candidates()
+ assert grounded in candidates
+ assert not any(_action_key(c, kwargs["targets"]) == failed for c in candidates)
+ return Ranking(grounded, [], [], 0, 0, 1, True)
+
+ s.components.selector.rank = rank
+ assert s.act(obs2).action_type == "click"
+
+
+def test_unrelated_animation_does_not_reset_semantic_no_effect(tmp_path):
+ s = session()
+ s.begin("Edit", requirements())
+ obs = make_observation(tmp_path, 0)
+ with pytest.raises(ObservationRequired):
+ s.act(obs)
+ s.act(obs)
+ for step, color in [(1, "black"), (2, "white")]:
+ obs = make_observation(tmp_path, step, receipt=gui_receipt())
+ Image.new("RGB", (64, 64), color).save(obs.screenshot.path)
+ assert s.act(obs).kind == "gui"
+ assert s.goal.no_effect == 2
+ assert all(h["observed_change"] and not h["relevant_change"] for h in s.goal.history)
+
+
+def test_unicode_codegen_uses_utf8_clipboard_not_ascii_typing():
+ text = "Organization Change — 组织调整"
+ action = GuiAction("type", {"text": text, "input_mode": "replace"})
+ gui = Mock()
+ with patch("subprocess.run") as run, patch("time.sleep"):
+ exec(gui_code(action), {"pyautogui": gui})
+ assert run.call_args.kwargs["input"] == text.encode("utf-8")
+ assert "UTF8_STRING" in run.call_args.args[0]
+ gui.write.assert_not_called()
+ gui.hotkey.assert_any_call("ctrl", "v")
+
+
+def test_partial_collection_extends_without_overwriting_or_claiming_complete():
+ memory = source_memory()
+ prepare(memory, fact(value="Partial options", entries={"1": "Humanities"}, complete=False))
+ assert not memory._facts_ready(memory.work_items["R1:unit:1"])
+ prepare(memory, fact(value="Full options", entries={"2": "Social Sciences"}, complete=True))
+ assert memory.facts["quota"]["entries"] == {"1": "Humanities", "2": "Social Sciences"}
+ assert memory.work_items["R1:prepare"]["status"] == "done"
+ memory.start(plan(), 2)
+ memory.apply(result(unit_met=False, facts=[fact(value="Changed", entries={"1": "Wrong"})]), {"A2": {}})
+ assert memory.facts["quota"]["entries"]["1"] == "Humanities"
+ assert memory.last_rejections[0]["reason"] == "source_update_requires_acquisition_stage"
+
+
+def test_preparation_targets_one_unit_not_all_materials():
+ req = requirements()[0]
+ req.update(units=["Save quota page", "Save sales page"], unit_fact_keys=[["quota"], ["sales"]],
+ facts_needed=[{"key": "quota", "description": "Quota"},
+ {"key": "sales", "description": "Sales"}])
+ memory = TaskMemory("Edit", [req])
+ memory.start({**plan(), "work_item_id": "R1:prepare", "purpose": "prepare"}, 0)
+ assert memory.active_stage["prepares_unit"] == "R1:unit:1"
+ assert memory.active_stage["fact_targets"] == [{"key": "quota", "description": "Quota"}]
+
+
+def test_snapshot_ids_and_ancestry_are_not_a_new_ui_state():
+ from dataclasses import replace
+
+ from tandem.runtime.goal_stage import _state_key
+
+ target = A11yTargetBuilder().build('''''', "one")[0]
+ changed_path = replace(target, snapshot_id="two", semantic_key="changed tab title|Open")
+ assert _state_key([target]) == _state_key([changed_path])
+
+
+def test_action_buttons_are_not_displaced_by_file_metadata_cells():
+ tree = ''
+ tree += ''.join(f''
+ for n in range(45))
+ tree += ''
+ targets = A11yTargetBuilder().build(tree, "s", 128, context="Open Image")
+ space = ActionCandidateBuilder().build(targets, context="Open Image")
+ button = next(t for t in targets if t.name == "Open")
+ assert any(c.target_id == button.id for c in space.target_heads["click"])
diff --git a/tests/harness/test_goal_progress.py b/tests/harness/test_goal_progress.py
new file mode 100644
index 0000000..7dd5cb4
--- /dev/null
+++ b/tests/harness/test_goal_progress.py
@@ -0,0 +1,302 @@
+"""Synthetic regressions for partial delivery, field binding and semantic navigation stalls."""
+
+import json
+from unittest.mock import Mock
+
+import pytest
+from PIL import Image
+
+from tandem.gui.compile import gui_action
+from tandem.gui.goal_candidates import goal_candidates, input_frontier
+from tandem.gui.grounding import ground_visual
+from tandem.memory.context import planning_context
+from tandem.memory.state import TaskMemory
+from tandem.models.ranking import Ranking
+from tandem.runtime.goal_stage import ObservationRequired, _document_mismatch, _navigation_key, _target_effect
+from tandem.verification.goal_verifier import verify_goal
+
+from .conftest import gui_receipt, make_observation
+from .test_goal_execution import _space
+from .test_goal_memory import fact, prepare, session, source_memory
+from .test_goal_stage import plan, requirements, result
+
+
+def test_partial_sources_allow_delivery_without_claiming_source_or_unit_complete():
+ memory = source_memory()
+ prepare(memory, fact(value="Known first section", complete=False))
+ item = planning_context(memory)["ready_work_items"][0]
+ assert item["id"] == "R1:unit:1" and item["can_start_partial"]
+ assert item["next_acquisition"]["missing_facts"][0]["key"] == "quota"
+ memory.start({**plan(), "completion_criteria": "Known first section saved"}, 1)
+ memory.apply(result(unit_met=False, delivery_state="saved", remaining=["Read remaining section"]),
+ {"A2": {}})
+ assert memory.work_items[item["id"]]["delivery_state"] == "saved"
+ assert memory.work_items[item["id"]]["status"] == "pending"
+ assert memory.work_items["R1:prepare"]["status"] != "done"
+ assert not memory.facts["quota"]["complete"]
+ memory.start({**plan(), "purpose": "prepare"}, 2)
+ assert memory.active_stage["fact_targets"][0]["key"] == "quota"
+
+
+def test_explicit_dependencies_still_apply_and_preparation_stays_with_selected_unit():
+ req = requirements()[0]
+ req.update(units=["First section", "Second section"], unit_fact_keys=[["first"], ["second"]],
+ facts_needed=[{"key": key, "description": key} for key in ("first", "second")])
+ memory = TaskMemory("Edit", [req])
+ second = memory.work_items["R1:unit:2"]
+ second["depends_on"] = ["R1:unit:1"]
+ assert second not in memory.ready_items()
+ second["depends_on"] = []
+ memory.start({**plan(), "work_item_id": second["id"], "purpose": "prepare"}, 0)
+ assert memory.active_stage["prepares_unit"] == second["id"]
+ assert memory.active_stage["fact_targets"][0]["key"] == "second"
+
+
+FORM = '''
+ file:///home/user/research-form.html?type=paper
+
+
+
+ '''
+
+
+def test_text_binds_to_observed_label_instead_of_unrelated_focus():
+ targets, space = _space(FORM, [{"label": "Article title", "target_hint": "Title",
+ "text": "Observed source title", "input_mode": "replace"}])
+ choices = space.target_heads["type_text"]
+ assert len(choices) == 1
+ target = next(t for t in targets if t.id == choices[0].target_id)
+ assert target.name == "Title *" and not target.focused
+ action = gui_action(choices[0], target)
+ assert action.parameters["click"] == [300, 115]
+ pending = {"operation": "type_text", "field_label": "Title *", "input_mode": "replace",
+ "expected_text": "Observed source title"}
+ wrong = FORM.replace('size="(200,30)"/>\n