diff --git a/README.md b/README.md
index 740d3a0..abf90a9 100644
--- a/README.md
+++ b/README.md
@@ -336,4 +336,6 @@ python -m http.server 8899 --directory demo/site # 本地预览
默认保持逐步规划;启用 `--execution-mode stage` 后,由慢模型规划阶段、快模型连续选择动作,
规则验证优先,仅在必要时调用模型验证或重规划。
+`--execution-mode goal` 可启用带任务记忆的局部连续执行,阶段结束再独立 Verify;
+保留 A11y/CDP 语义融合及 JEV 服务恢复,默认 step/stage 行为不变。
配置、执行边界、开发入口和验收方法见 [阶段执行说明](docs/STAGE_EXECUTION.md)。
diff --git a/docs/RESULT_DIR_CONTRACT.md b/docs/RESULT_DIR_CONTRACT.md
index 6867654..5cda70c 100644
--- a/docs/RESULT_DIR_CONTRACT.md
+++ b/docs/RESULT_DIR_CONTRACT.md
@@ -39,6 +39,12 @@ OSWorld 2.1 显式配置替代 judge/用户模拟器时,`code_version.json.res
| `result.txt` | 分数(`score\n`) | 同上 |
| `episode.json` | `{task_id, harness, stop_reason, steps, status(scored\|failed_unscored), score, exception_type, error?, error_traceback?}` | episode 终态(必写)。`error`/`error_traceback` 仅异常题有:异常消息与栈尾几行(脱敏)——只记类型事后没法定位 |
| `timing.json` | `{wall_seconds, status}` | episode 终态 |
+| `task_memory.json`(goal) | 最新任务记忆:要求、事实、资料覆盖、交付单元、成果、失败与当前阶段 | goal 事件排水及终态 |
+| `evidence/`(goal) | 阶段验证引用的截图/CLI 全文正本;临时目录清理后仍可读取 | 阶段验收/记忆落盘 |
+
+`timing.json` 另外提供 `environment_seconds`、`model_usage_by_role`、`fast_model`,分别记录环境
+分段时间、逐角色请求时间/token、排序次数/耗时/图像使用次数。缺失 usage 不补零冒充已测值,
+另记 `calls_missing_usage`。原字段及 `browser/` 正本规则保持不变。
## 步号语义(demo 侧对齐依据)
@@ -63,7 +69,9 @@ OSWorld 2.1 显式配置替代 judge/用户模拟器时,`code_version.json.res
|------|--------|---------|
| `task_requirements` | verifier.task_requirements | episode 开始时抽取任务需求(1-12 条) |
| `plan_step` | planner.plan_step | 每步规划(截图+可见控件+需求+近期动作→下一步决策) |
-| `plan_stage` | stage_planner.plan_stage | 仅 stage 模式:初始阶段、阶段完成或请求修复时生成阶段计划 |
+| `plan_stage` | stage_planner.plan_stage / goal_planner.plan_goal | stage/goal 的阶段开始或修复;goal 允许读取已存资料后继续规划 |
+| `verify_stage` | goal_verifier.verify_goal | goal 阶段结束时独立验证或提取资料增量;明确执行失败的本地检查不算模型调用 |
+| `ground_visual` | grounding.ground_visual | goal 按需生成当前截图的有限候选,不直接执行慢模型坐标 |
| `verify_observation` | verifier.verify_observation | 每步动作执行后独立验证(BEFORE/AFTER 截图→achieved/not_achieved/uncertain)|
`verify_observation` 不信任 planner 的自我宣称:它拿前后截图+预期变化独立裁决,驱动 recent 历史、no_action 连击计数、toggle 检查、以及终态双确认(terminal_claim 模式下逐条核验 requirements + 输出文件存在性)。
@@ -80,6 +88,12 @@ OSWorld 2.1 显式配置替代 judge/用户模拟器时,`code_version.json.res
| `stage_local` | 消费已生成的阶段计划;含 stage_goal、milestone 和当帧重绑定后的 plan,不代表新的模型调用 |
| `stage_progress` | 里程碑推进,含 stage_goal、milestone、验证 source |
| `stage_replan` | 阶段修复请求,含原因及当前里程碑;target_unbound 时另含 `unbound_proposals`、`available_targets`、`recovery_scopes`;不等于调用已经完成 |
+| `task_memory` / `memory_update_rejected` | goal 的完整记忆快照/拒绝理由;不是规划模型自行宣称的完成 |
+| `goal_action` | 真实回执、当前控件、画面变化与局部效果;effect_verified=false,不等于业务验收通过 |
+| `stage_verified` | goal 的阶段裁决、证据引用、结束原因与用时,区分阶段成功和单元完成 |
+| `observation_refresh` | 刷新观察而未执行动作,action_replayed=false;不增加动作步数 |
+| `recovery` / `goal_stalled` | goal 的视觉候选补全/无可执行进展终止原因 |
+| `environment_timing` | setup/observe/execute/export/evaluate 分段耗时 |
| `action`(kind 字段) | runner 写的执行记录 `{step, kind: "action", action: {…协议动作…}}` |
stage 模式的 planner 事件只在真实规划调用时写入,增加 mode、reason、seconds;
diff --git a/infra/config.example.json b/infra/config.example.json
index a652093..9dc0486 100644
--- a/infra/config.example.json
+++ b/infra/config.example.json
@@ -63,7 +63,12 @@
},
"execution": {
"mode": "step",
- "stage_max_actions": 24
+ "stage_max_actions": 24,
+ "verify_max_tokens": 16384,
+ "verify_reasoning_effort": null,
+ "verify_max_chars": 48000,
+ "image_quality": 85,
+ "max_request_bytes": 2097152
},
"budget": {
"max_steps": 30,
diff --git a/pyproject.toml b/pyproject.toml
index 1210a05..9024978 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,8 +7,8 @@ name = "tandem"
version = "0.1.0"
description = "Tandem:快慢双模 CUA harness(主体)+ tandem_eval 集群测评薄适配层"
requires-python = ">=3.10"
-# 运行时依赖白名单见 AGENTS.md。当前为空:M0 尚无消费者;Pillow 在 M1 截图处理落地时加入。
-dependencies = []
+# 截图处理使用仓库白名单内的 Pillow。
+dependencies = ["Pillow"]
[project.optional-dependencies]
dev = ["pytest>=7", "ruff>=0.4"]
diff --git a/scripts/run_experiment.sh b/scripts/run_experiment.sh
index 9705f10..5f4729f 100755
--- a/scripts/run_experiment.sh
+++ b/scripts/run_experiment.sh
@@ -12,6 +12,8 @@ SLOW_MODEL="kimi-k3"
SLOW_KEY_IDS="1,2,3,4"
MAX_STEPS=500
MAX_WALL=14400
+EXECUTION_MODE=stage
+STAGE_MAX_ACTIONS=10
DRY_RUN=0
usage() {
@@ -20,6 +22,7 @@ Usage:
bash scripts/run_experiment.sh \
--experiment NAME --tasks ID1,ID2,... [--concurrency N] \
[--slow-model NAME] [--slow-key-ids 1,2,3,4] \
+ [--execution-mode step|stage|goal] [--stage-max-actions N] \
[--max-steps N] [--max-wall SECONDS] [--dry-run]
Experiment names use lowercase letters, digits and hyphens only.
@@ -36,6 +39,8 @@ while [[ $# -gt 0 ]]; do
--slow-key-ids) SLOW_KEY_IDS="$2"; shift 2 ;;
--max-steps) MAX_STEPS="$2"; shift 2 ;;
--max-wall) MAX_WALL="$2"; shift 2 ;;
+ --execution-mode) EXECUTION_MODE="$2"; shift 2 ;;
+ --stage-max-actions) STAGE_MAX_ACTIONS="$2"; shift 2 ;;
--dry-run) DRY_RUN=1; shift ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
@@ -50,6 +55,10 @@ if [[ -z "$TASKS" || "$CONCURRENCY" -lt 1 || "$MAX_STEPS" -lt 1 || "$MAX_WALL" -
usage >&2
exit 2
fi
+if [[ ! "$EXECUTION_MODE" =~ ^(step|stage|goal)$ || "$STAGE_MAX_ACTIONS" -lt 1 ]]; then
+ usage >&2
+ exit 2
+fi
EXPERIMENT_ROOT="${TANDEM_EXPERIMENT_ROOT:-$REPO/rjob_logs}"
EXPERIMENT_DIR="$EXPERIMENT_ROOT/$EXPERIMENT"
@@ -77,6 +86,7 @@ print(json.loads(Path(sys.argv[1]).read_text())["log"]["results_root"])
PY
)
export STAGING EXPERIMENT DRIVER_DIR TASKS CONCURRENCY SLOW_MODEL SLOW_KEY_IDS MAX_STEPS MAX_WALL
+export EXECUTION_MODE STAGE_MAX_ACTIONS
python3 - <<'PY'
import json
import os
@@ -93,6 +103,8 @@ args = {
"slow_key_ids": [int(item) for item in os.environ["SLOW_KEY_IDS"].split(",")],
"max_steps": int(os.environ["MAX_STEPS"]),
"max_wall_seconds": float(os.environ["MAX_WALL"]),
+ "execution_mode": os.environ["EXECUTION_MODE"],
+ "stage_max_actions": int(os.environ["STAGE_MAX_ACTIONS"]),
"watch_timeout_seconds": float(os.environ["MAX_WALL"]) + 600,
"git_describe": subprocess.check_output(
["git", "describe", "--always", "--dirty"], text=True
@@ -124,12 +136,12 @@ run_task() {
local driver_log="${DRIVER_DIR}/${task}.driver.log"
local rc=0
- tandem-eval rjob run \
+ env PYTHONPATH=src python3 -m tandem_eval rjob run \
--run-id "$run_id" \
--benchmark osworld21 \
--tasks "$task" \
- --execution-mode stage \
- --stage-max-actions 10 \
+ --execution-mode "$EXECUTION_MODE" \
+ --stage-max-actions "$STAGE_MAX_ACTIONS" \
--slow-model "$SLOW_MODEL" \
--slow-key-ids "$SLOW_KEY_IDS" \
--max-steps "$MAX_STEPS" \
diff --git a/src/tandem/components.py b/src/tandem/components.py
index 3df6554..54709d0 100644
--- a/src/tandem/components.py
+++ b/src/tandem/components.py
@@ -1,6 +1,6 @@
"""组件契约(依赖注入边界,无插件框架、无隐藏分发)。
-在旧九组件上增加 semantic_enricher,统一 slow planner 与 fast selector 看到的元素语义。
-grounder(visual fallback)默认 None,session 侧判空跳过。compile_gui 产出协议动作。"""
+semantic_enricher 统一 planner 与 selector 的元素语义;goal 额外注入任务记忆、阶段验证与
+按需视觉补全。step/stage 装配不变。compile_gui 只产出协议动作,不调用 VM。"""
from __future__ import annotations
@@ -11,12 +11,16 @@
from .config import SessionConfig
from .gui.candidates import ActionCandidateBuilder
from .gui.compile import gui_action
+from .gui.grounding import ground_visual
from .gui.proposals import focused_candidates
from .gui.semantics import enrich_targets
from .gui.targets import A11yTargetBuilder
+from .memory.state import TaskMemory
from .models.jev import JevOmniPolicy
+from .planning.goal_planner import goal_requirements, plan_goal
from .planning.planner import plan_step
from .planning.stage_planner import plan_stage
+from .verification.goal_verifier import verify_goal
from .verification.verifier import task_requirements, verify_observation
@@ -32,6 +36,8 @@ class Components:
focus_candidates: Callable
grounder: Callable | None
compile_gui: Callable
+ memory_factory: Callable | None = None
+ stage_verifier: Callable | None = None
def build_default(*, client, endpoint: str, config: SessionConfig,
@@ -42,21 +48,26 @@ def build_default(*, client, endpoint: str, config: SessionConfig,
selector = KevPolicy(endpoint, timeout_seconds=config.fast_timeout_seconds)
else:
selector = JevOmniPolicy(endpoint, timeout_seconds=config.fast_timeout_seconds,
- image_stage=image_stage, endpoint_provider=endpoint_provider)
+ image_stage=image_stage, endpoint_provider=endpoint_provider,
+ joint_actions=config.execution_mode == "goal")
return Components(
planner=partial(
+ plan_goal if config.execution_mode == "goal" else
plan_stage if config.execution_mode == "stage" else plan_step,
client,
allow_cli=config.allow_cli and config.route_policy != "gui",
route_policy=config.route_policy,
),
verifier=partial(verify_observation, client),
- requirements=partial(task_requirements, client),
+ requirements=partial(
+ goal_requirements if config.execution_mode == "goal" else task_requirements, client),
selector=selector,
target_builder=A11yTargetBuilder(),
semantic_enricher=enrich_targets,
candidate_builder=ActionCandidateBuilder(),
focus_candidates=focused_candidates,
- grounder=None,
+ grounder=partial(ground_visual, client) if config.execution_mode == "goal" else None,
compile_gui=gui_action,
+ memory_factory=TaskMemory if config.execution_mode == "goal" else None,
+ stage_verifier=partial(verify_goal, client) if config.execution_mode == "goal" else None,
)
diff --git a/src/tandem/config.py b/src/tandem/config.py
index 5786acc..9d4a2c3 100644
--- a/src/tandem/config.py
+++ b/src/tandem/config.py
@@ -18,15 +18,20 @@ class SessionConfig:
execution_mode: str = "step"
stage_max_actions: int = 24
slow_max_tokens: int = 16384
+ slow_verify_max_tokens: int = 16384
+ slow_verify_reasoning_effort: str | None = None
+ stage_verify_max_chars: int = 48000
slow_timeout_seconds: float = 120
slow_direct_api: bool = False
slow_reasoning_effort: str | None = None # None=服务端默认;off=enable_thinking:false
+ slow_image_quality: int = 85
+ slow_max_request_bytes: int = 2 * 1024 * 1024
fast_backend: str = "jev" # "jev"(qwen-jev-choice /rank)| "kev"(Kev System One)
fast_timeout_seconds: float = 300
def __post_init__(self):
- if self.execution_mode not in {"step", "stage"}:
- raise ValueError("execution_mode must be step or stage")
+ if self.execution_mode not in {"step", "stage", "goal"}:
+ raise ValueError("execution_mode must be step, stage or goal")
if type(self.stage_max_actions) is not int or self.stage_max_actions < 1:
raise ValueError("stage_max_actions must be a positive integer")
if self.route_policy not in {"auto", "gui", "cli"}:
@@ -37,3 +42,5 @@ def __post_init__(self):
raise ValueError("fast_backend must be jev or kev")
if self.slow_reasoning_effort not in {None, "off", "low", "medium", "high", "max"}:
raise ValueError("slow_reasoning_effort must be null/off/low/medium/high/max")
+ if self.slow_verify_reasoning_effort not in {None, "off", "low", "medium", "high", "max"}:
+ raise ValueError("slow_verify_reasoning_effort must be null/off/low/medium/high/max")
diff --git a/src/tandem/gui/compile.py b/src/tandem/gui/compile.py
index c8be177..b03b040 100644
--- a/src/tandem/gui/compile.py
+++ b/src/tandem/gui/compile.py
@@ -17,10 +17,14 @@ def gui_action(
) -> GuiAction | WaitAction | None:
operation = winner.operation
metadata = winner.metadata
- if operation in {"click", "double_click"} and target is not None:
+ if operation in {"click", "double_click", "right_click"} and (
+ target is not None or metadata.get("visual_point")
+ ):
+ point = metadata.get("visual_point") if target is None else [
+ target.x + target.width // 2, target.y + target.height // 2]
return GuiAction(
action_type=operation,
- parameters={"x": target.x + target.width // 2, "y": target.y + target.height // 2},
+ parameters={"x": point[0], "y": point[1]},
target_id=winner.target_id,
description=winner.description,
)
@@ -56,7 +60,7 @@ def gui_action(
description=winner.description,
)
if operation == "type_text":
- click = None
+ click = metadata.get("visual_point")
press_f2 = False
if target is not None and not metadata.get("focused_input"):
click = [target.x + target.width // 2, target.y + target.height // 2]
diff --git a/src/tandem/gui/goal_candidates.py b/src/tandem/gui/goal_candidates.py
new file mode 100644
index 0000000..6b1a8b7
--- /dev/null
+++ b/src/tandem/gui/goal_candidates.py
@@ -0,0 +1,187 @@
+"""Current-observation action space for a whole GUI subgoal; no planner click whitelist."""
+
+import json
+import re
+import xml.etree.ElementTree as ET
+
+from .candidates import ActionCandidate
+from .focus import input_and_menu_context
+from .targets import _attribute, _local, _pair, _state
+
+
+def matches_field(hint, label):
+ """Match an intended field to its observed label, not an arbitrary focused control."""
+ def words(value):
+ return set(re.findall(r"\w+", value.casefold())) - {"the", "a", "field", "required"}
+
+ observed = words(label)
+ return bool(observed) and observed <= words(hint) and not label.startswith(("Unnamed ", "Focused "))
+
+
+def scroll_point(tree, viewport):
+ """Anchor wheel events inside the observed foreground window, not the desktop center."""
+ windows = []
+ for element in ET.fromstring(tree or "").iter():
+ if _local(element.tag) not in {"frame", "dialog"} or _state(element, "active") is not True:
+ continue
+ position = _pair(_attribute(element, "screencoord"))
+ size = _pair(_attribute(element, "size"))
+ if position and size and size[0] > 0 and size[1] > 0:
+ windows.append((*position, *size))
+ x, y, width, height = windows[-1] if windows else (0, 0, *viewport)
+ return (max(0, min(viewport[0] - 1, x + width * 3 // 5)),
+ max(0, min(viewport[1] - 1, y + height // 2)))
+
+
+def foreground_tree(raw_tree):
+ """GNOME's visible app menu disambiguates stale ACTIVE flags on background windows."""
+ root = ET.fromstring(raw_tree)
+ apps = list(root)
+ shell = next((a for a in apps if a.get("name") == "gnome-shell"), None)
+ if shell is None:
+ return raw_tree
+ overview = [e for e in shell.iter() if e.get("name") == "Overview"]
+ search_visible = any(
+ _state(e, "editable") is True and _state(e, "showing") is True
+ and _pair(_attribute(e, "screencoord")) and _pair(_attribute(e, "size"))
+ for panel in overview for e in panel.iter())
+ if search_visible:
+ # X11 still names the old window while GNOME's full-screen overview owns input.
+ for app in apps:
+ if app is not shell:
+ root.remove(app)
+ shell.set("active", "true")
+ root.set("foreground_title", "GNOME Overview")
+ root.set("foreground_class", '["gnome-shell"]')
+ return ET.tostring(root, encoding="unicode")
+ if root.get("foreground_class"):
+ def normalize(value):
+ return re.sub(r"[^a-z0-9]", "", value.casefold())
+
+ def active(element, value):
+ key = next((k for k in element.attrib if _local(k) == "active"), "active")
+ element.set(key, "true" if value else "false")
+
+ classes = [normalize(v) for v in json.loads(root.get("foreground_class")) if v]
+ title = root.get("foreground_title", "")
+ for app in apps:
+ matching = (any(normalize(app.get("name", "")).endswith(c) for c in classes)
+ or bool(title) and any(e.get("name") == title for e in app.iter()
+ if _local(e.tag) in {"frame", "dialog"}))
+ if app is not shell and not matching:
+ root.remove(app)
+ elif app is not shell:
+ windows = [e for e in app.iter() if _local(e.tag) in {"frame", "dialog"}]
+ named = [e for e in windows if title and e.get("name") == title]
+ if named:
+ active(app, False)
+ for window in windows:
+ active(window, window in named)
+ return ET.tostring(root, encoding="unicode")
+ names = []
+ for element in shell.iter():
+ attrs = {k.rsplit("}", 1)[-1]: v for k, v in element.attrib.items()}
+ position = re.findall(r"-?\d+", attrs.get("screencoord", ""))
+ if (element.tag.rsplit("}", 1)[-1] == "menu" and attrs.get("name")
+ and attrs.get("showing") == "true" and len(position) == 2
+ and 0 <= int(position[1]) < 32):
+ names.append(attrs["name"].casefold())
+ matches = [app for app in apps if app is not shell and any(
+ name in e.get("name", "").casefold() for e in app.iter()
+ if e.tag.rsplit("}", 1)[-1] in {"frame", "dialog"} for name in names)]
+ if len(matches) != 1:
+ return raw_tree
+ for app in apps:
+ if app is not shell and app is not matches[0]:
+ root.remove(app)
+ return ET.tostring(root, encoding="unicode")
+
+
+def goal_candidates(space, targets, tree, inputs, viewport=(1920, 1080)):
+ _, menus = input_and_menu_context(tree or "")
+ fields = [t for t in targets if t.role in {"text", "textbox", "entry", "searchbox", "spin_button"}]
+ typed = []
+ for index, value in enumerate(inputs):
+ hint = value.get("target_hint", value["label"])
+ matching = [t for t in fields if matches_field(hint, t.name)]
+ if len(matching) != 1:
+ continue # Ambiguous/unlabelled fields need an observed click or visual grounding first.
+ target = matching[0]
+ if (target.value == value["text"] or value["input_mode"] == "insert"
+ and value["text"] in target.value):
+ continue
+ typed.append(ActionCandidate(
+ id=f"GOAL_TEXT:{index}", operation="type_text", source="goal_input", target_id=target.id,
+ description=f"{value['input_mode']} the observed field {target.name!r} with the "
+ f"supplied {value['label']}: {value['text']!r}",
+ metadata={"text": value["text"], "input_mode": value["input_mode"],
+ "target_hint": hint, "field_label": target.name,
+ "semantic_key": target.semantic_key},
+ ))
+ space.target_heads["type_text"] = typed
+ unnamed_entries = [t for t in targets if t.name.startswith(("Unnamed ", "Focused "))
+ and t.role in {"text", "textbox", "entry", "searchbox"}][:2]
+ present = {c.target_id for c in space.target_heads["click"]}
+ space.target_heads["click"] = [ActionCandidate(
+ id="FOCUS:" + t.id, operation="click", target_id=t.id, source="a11y",
+ description=f"Focus the observed {t.name!r} in {' / '.join(t.parent_context)} before typing.",
+ metadata={"semantic_key": t.semantic_key}) for t in unnamed_entries if t.id not in present
+ ] + space.target_heads["click"]
+ native_files = [t for t in targets if t.role in {"canvas", "label"}
+ and "nautilus" in (t.application + " ".join(t.parent_context)).lower()][:4]
+ space.target_heads["double_click"].extend(ActionCandidate(
+ id="OPEN:" + t.id, operation="double_click", target_id=t.id, source=t.source,
+ description=f"Open the observed file-manager item {t.name!r}.",
+ metadata={"semantic_key": t.semantic_key}) for t in native_files)
+ sliders = [t for t in targets if t.role == "slider"][:2]
+ space.target_heads["set_slider"] = [ActionCandidate(
+ id=f"SLIDER:{t.id}:{fraction}", operation="set_slider", target_id=t.id,
+ description=f"Set {t.name!r} to {fraction:.0%} of its range"
+ + (f" (value {t.range_min + fraction * (t.range_max - t.range_min):.2f})"
+ if t.range_min is not None and t.range_max is not None else "")
+ + f"; current observed value: {t.value or 'unknown'}.",
+ metadata={"fraction": fraction, "semantic_key": t.semantic_key})
+ for t in sliders for fraction in (0.1, 0.25, 0.5, 0.75, 0.9)
+ if (not t.value or t.range_min is None or t.range_max is None
+ or abs(float(t.value) - (t.range_min + fraction * (t.range_max - t.range_min)))
+ > max(1e-6, (t.range_max - t.range_min) * 0.02))]
+ # A slider's default center click is ambiguous and was repeatedly seeking to the same midpoint.
+ slider_ids = {t.id for t in sliders}
+ space.target_heads["click"] = [c for c in space.target_heads["click"] if c.target_id not in slider_ids]
+ space.target_heads["right_click"] = [ActionCandidate(
+ id="RIGHT:" + c.target_id, operation="right_click", source=c.source, target_id=c.target_id,
+ description="Open the context menu for " + c.description,
+ metadata=dict(c.metadata)) for c in space.target_heads["click"][:8]]
+ sx, sy = scroll_point(tree, viewport)
+ for candidate in space.target_heads["scroll"]:
+ candidate.metadata.update(scroll_x=sx, scroll_y=sy)
+ for menu in menus[:16]:
+ space.target_heads["keypress"].append(ActionCandidate(
+ id="MENU_KEY:" + menu["path"], operation="keypress", source="a11y_menu",
+ description="Open the observed menu path " + menu["path"],
+ metadata={"key_sequence": menu["key_sequence"]}))
+ space.target_heads["keypress"].extend(ActionCandidate(
+ id="GOAL_KEY:" + "+".join(keys), operation="keypress", source="global",
+ description=description, metadata={"key_sequence": [keys]}) for keys, description in [
+ (["ctrl", "s"], "Save the current document."),
+ (["ctrl", "o"], "Open the application's file chooser for an existing document."),
+ (["alt", "tab"], "Switch away from the foreground window to another open application."),
+ (["shift", "f10"], "Open the focused object's context menu."),
+ (["ctrl", "l"], "Focus the browser or native file-dialog location field."),
+ (["ctrl", "f"], "Open the current application's Find field."),
+ (["shift", "tab"], "Move focus to the previous field."),
+ (["down"], "Move down in the current selection or menu."),
+ (["up"], "Move up in the current selection or menu."),
+ (["right"], "Move right in the current selection, calendar or menu."),
+ (["left"], "Move left in the current selection, calendar or menu."),
+ ])
+ space.controls = {k: v for k, v in space.controls.items()
+ if k in {"wait", "done", "visual_fallback", "return_to_planner"}}
+ space.controls["done"] = ActionCandidate(
+ id="STAGE_DONE", operation="done", source="stage_control",
+ description="The current subtask's completion criteria are satisfied. Request ONE independent "
+ "stage verification; this does not finish the original task.")
+ space.controls["return_to_planner"].description = (
+ "The current subtask needs unavailable information, new reasoning or an unsupported operation. "
+ "Return control with existing progress preserved; do not repeat an ineffective action.")
+ return space
diff --git a/src/tandem/gui/grounding.py b/src/tandem/gui/grounding.py
new file mode 100644
index 0000000..b7a65a2
--- /dev/null
+++ b/src/tandem/gui/grounding.py
@@ -0,0 +1,66 @@
+"""Existing screenshot-grounded target fallback, reused without changing the JEV service."""
+
+import base64
+import json
+
+from PIL import Image
+
+from .candidates import ActionCandidate
+from .goal_candidates import matches_field
+
+
+def ground_visual(client, instruction, plan, screenshot):
+ with Image.open(screenshot) as image:
+ width, height = image.size
+ proposals = plan["action_proposals"]
+ value = client.json_request([
+ {"role": "system", "content": (
+ "Ground the immediate GUI intent in the CURRENT screenshot. Propose up to four distinct "
+ "visible target alternatives, never a screen grid or a sequence. Do not execute anything. "
+ "Return JSON {candidates:[{operation,target,bounds,text,input_mode,target_hint}]}. "
+ "Operations: click, double_click, right_click, type_text, scroll_up, scroll_down. "
+ "bounds=[x,y,width,height] in "
+ "ORIGINAL screenshot pixels. Ground only foreground unobscured targets. For type_text copy exact "
+ "text/input_mode AND target_hint from ONE supplied input, and ground the visible field named "
+ "by that hint. target must be its observed label/identity, not the text to enter. Never put "
+ "unrelated text in the currently focused field; otherwise click to focus first. Use [] if no "
+ "required target is visible. Do not invent targets or completion evidence."
+ )},
+ {"role": "user", "content": [
+ {"type": "text", "text": json.dumps({"task": instruction, "local_plan": plan,
+ "screenshot_size": [width, height]}, ensure_ascii=False)},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,"
+ + base64.b64encode(screenshot.read_bytes()).decode("ascii")}},
+ ]},
+ ], lambda v: isinstance(v.get("candidates"), list)
+ and all(isinstance(i, dict) for i in v["candidates"]), role="ground_visual")
+ allowed_text = {(p.get("text"), p.get("input_mode"), p.get("target_hint", p.get("label", "")))
+ for p in proposals if p.get("operation") == "type_text"}
+ candidates = []
+ for item in value["candidates"][:4]:
+ operation, box = item.get("operation"), item.get("bounds")
+ if operation not in {"click", "double_click", "right_click", "type_text", "scroll_up", "scroll_down"}:
+ continue
+ if not isinstance(box, list) or len(box) != 4 or any(type(v) is not int for v in box):
+ continue
+ x, y, w, h = box
+ if not (0 <= x < x + w <= width and 0 <= y < y + h <= height):
+ continue
+ point = [x + w // 2, y + h // 2]
+ metadata = {"visual_point": point, "visual_bounds": box}
+ if operation == "type_text":
+ if ((item.get("text"), item.get("input_mode"), item.get("target_hint")) not in allowed_text
+ or not matches_field(item.get("target_hint", ""), item.get("target", ""))):
+ continue
+ metadata.update(text=item["text"], input_mode=item["input_mode"],
+ target_hint=item["target_hint"], field_label=item["target"])
+ name = f"VISUAL:{operation}:{x}:{y}:{w}:{h}"
+ if operation.startswith("scroll_"):
+ name = f"{operation.upper()}:{name}"
+ operation = "scroll"
+ metadata.update(scroll_x=point[0], scroll_y=point[1])
+ candidates.append(ActionCandidate(
+ id=name, operation=operation, source="visual_grounder", metadata=metadata,
+ description=f"{item['operation']} screenshot-visible {item.get('target', 'control')!r} "
+ f"at box {box}." + (f" Text: {metadata['text']!r}." if "text" in metadata else "")))
+ return candidates
diff --git a/src/tandem/gui/targets.py b/src/tandem/gui/targets.py
index 0f7bd8a..9b9f915 100644
--- a/src/tandem/gui/targets.py
+++ b/src/tandem/gui/targets.py
@@ -33,6 +33,8 @@ class UITarget:
scope: str = "native"
label_source: str = "a11y"
operations: tuple[str, ...] = ()
+ range_min: float | None = None
+ range_max: float | None = None
def compact(self) -> dict:
return {key: value for key, value in asdict(self).items() if value not in (None, "", (), False)}
@@ -100,6 +102,32 @@ def _attribute(element: ET.Element, name: str) -> str:
return ""
+def _number(element, name):
+ value = _attribute(element, name)
+ return float(value) if re.fullmatch(r"-?\d+(?:\.\d+)?", value) else None
+
+
+def _is_overlay(element):
+ return _local(element.tag) in {"dialog", "alert", "popup_menu"} or (
+ _local(element.tag) == "menu" and _attribute(element, "ispopup") in {"true", "auto", "menu"})
+
+
+def _field_labels(root):
+ labels = {}
+ for parent in root.iter():
+ preceding = ""
+ for child in parent:
+ if _local(child.tag) in {"label", "static"}:
+ preceding = str(child.get("name") or child.text or "").strip()
+ continue
+ fields = [e for e in child.iter() if _state(e, "editable") is True
+ or _local(e.tag) in {"entry", "textbox", "searchbox", "combo_box", "spin_button"}]
+ if preceding and len(fields) == 1:
+ labels[fields[0]] = preceding
+ preceding = ""
+ return labels
+
+
class A11yTargetBuilder:
def build(
self,
@@ -108,22 +136,40 @@ def build(
max_targets: int = 64,
infer_button_labels: bool = False,
include_sliders: bool = False,
+ context: str = "",
+ filter_occluded: bool = False,
) -> list[UITarget]:
root = ET.fromstring(raw_tree)
targets: list[UITarget] = []
+ field_labels = _field_labels(root) if filter_occluded else {}
+ target_nodes, overlays = {}, {}
+ if filter_occluded:
+ for node in root.iter():
+ if (_is_overlay(node) and _state(node, "showing") is True
+ and _state(node, "visible") is not False):
+ position, size = _pair(_attribute(node, "screencoord")), _pair(_attribute(node, "size"))
+ if position and size and min(size) > 0:
+ overlays.setdefault((*position, *size), set()).update(node.iter())
- def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name: str) -> None:
+ def visit(element, active, parents, app_name, modal=False, roles=(), file_manager=False):
active = active or _state(element, "active") is True
role = _local(element.tag)
+ modal = modal or _is_overlay(element)
raw_name = str(element.attrib.get("name") or "").strip()
name = raw_name
if role == "application":
app_name = name
label_source = "a11y" if name else "unresolved"
+ if not name and element in field_labels:
+ name, label_source = field_labels[element], "a11y_adjacent"
+ if (not name and filter_occluded and "gnome-shell" in parents
+ and "Overview" in parents and _state(element, "editable") is True):
+ name, label_source = "Desktop search", "a11y_context"
+ file_manager = file_manager or "nautilus" in app_name.lower() or "file chooser" in name.lower()
value = (
str(element.text or "").strip()
if role in {"text", "textbox", "entry", "searchbox", "spin_button"}
- else ""
+ else _attribute(element, "value") if role == "slider" else ""
)
position = _pair(_attribute(element, "screencoord"))
size = _pair(_attribute(element, "size"))
@@ -149,6 +195,9 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name:
and active
and (
any("file chooser" in item.lower() or item.lower() == "places" for item in parents)
+ or filter_occluded and file_manager and any(r in {
+ "list_item", "tree_item", "button", "push_button", "menu_item", "option", "link"}
+ for r in roles)
or bool(
re.search(
r"\b(?:monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b", name, re.I
@@ -158,7 +207,8 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name:
)
if (
(active or system_target)
- and (role in SUPPORTED_ROLES or safe_label or (include_sliders and role == "slider"))
+ and (role in SUPPORTED_ROLES or safe_label or (include_sliders and role == "slider")
+ or filter_occluded and (role == "image" or file_manager and role == "canvas"))
and (name or role in SUPPORTED_ROLES)
and interactive_text
and enabled is not False
@@ -168,7 +218,6 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name:
and size
and size[0] > 0
and size[1] > 0
- and len(targets) < max_targets
):
normalized_name = " ".join(name.lower().split())
parent_context = tuple(item for item in parents[-3:] if item)
@@ -198,10 +247,24 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name:
or _attribute(element, "placeholder-text")
),
label_source=label_source,
+ source="a11y_dialog" if filter_occluded and modal else "a11y",
+ range_min=_number(element, "min") if role == "slider" else None,
+ range_max=_number(element, "max") if role == "slider" else None,
)
)
+ target_nodes[targets[-1].id] = element
for child in element:
- visit(child, active, context[-4:], app_name)
+ visit(child, active, context[-4:], app_name, modal, (*roles, role), file_manager)
visit(root, False, (), "")
- return targets
+ if filter_occluded:
+ targets = [t for t in targets if not any(
+ target_nodes[t.id] not in nodes and x <= t.x + t.width // 2 < x + width
+ and y <= t.y + t.height // 2 < y + height
+ for (x, y, width, height), nodes in overlays.items())]
+ if context:
+ words = set(re.findall(r"\w+", context.lower()))
+ targets.sort(key=lambda t: (t.source == "a11y_dialog",
+ len(words & set(re.findall(r"\w+", t.name.lower()))), t.focused is True,
+ t.role in {"button", "push_button", "combo_box", "entry", "textbox"}), reverse=True)
+ return targets[:max_targets]
diff --git a/src/tandem/memory/__init__.py b/src/tandem/memory/__init__.py
new file mode 100644
index 0000000..e6c80da
--- /dev/null
+++ b/src/tandem/memory/__init__.py
@@ -0,0 +1 @@
+"""Task-scoped source facts, delivery progress and execution evidence."""
diff --git a/src/tandem/memory/context.py b/src/tandem/memory/context.py
new file mode 100644
index 0000000..d930597
--- /dev/null
+++ b/src/tandem/memory/context.py
@@ -0,0 +1,59 @@
+"""Compact role-specific views. Raw receipts/screenshots remain in the episode record."""
+
+
+def _fact_view(fact, limit=2400):
+ view = {k: v for k, v in fact.items() if k not in {"observations", "entries"}}
+ if len(view["value"]) > limit:
+ view["value"] = (view["value"][:limit // 2]
+ + "\n[excerpt; request read_memory for the stored content]\n"
+ + view["value"][-limit // 2:])
+ view["value_excerpt_only"] = True
+ return view
+
+
+def planning_context(memory):
+ ready = memory.ready_items()[:4]
+ keys = {f["key"] for item in ready for f in (
+ memory.preparation_target(item)[0]["facts_needed"] if item["kind"] == "prepare"
+ else item["facts_needed"])}
+ facts = [f for f in memory.facts.values() if f["scope"] == "task"]
+ auxiliary = {f["key"] for f in facts[-8:]}
+ visible = [_fact_view(f) for f in facts if f["key"] in keys | auxiliary]
+ return {"requirements": [{k: r[k] for k in ("id", "requirement", "kind", "status")}
+ for r in memory.requirements],
+ "ready_work_items": [{**item, "can_start_partial": item["kind"] == "deliver",
+ "next_acquisition": {
+ "unit": memory.preparation_target(item)[0]["goal"],
+ "missing_facts": memory.preparation_target(item)[1]}}
+ for item in ready],
+ "completed_work": [i["id"] for i in memory.work_items.values() if i["status"] == "done"],
+ "known_facts": visible,
+ "fact_index": [{**{k: f[k] for k in ("key", "status", "source_ref")},
+ "complete": f.get("complete", True)} for f in facts],
+ "historical_ui_notes": [f for f in memory.facts.values() if f["scope"] == "stage"][-6:],
+ "materials": [{"source_ref": m["source_ref"], "covered_parts": m["covered_parts"],
+ "summary_excerpt": m["summary"][:1800],
+ "summary_excerpt_only": len(m["summary"]) > 1800}
+ for m in list(memory.materials.values())[-8:]],
+ "artifacts": list(memory.artifacts.values()), "failures": memory.failures[-12:],
+ "execution_constraints": memory.execution_constraints[-12:],
+ "memory_update_rejections": memory.last_rejections,
+ "stages_without_progress": memory.stats["stages_without_progress"]}
+
+
+def execution_context(memory):
+ stage = memory.active_stage
+ item = memory.work_items[stage["work_item_id"]]
+ return {"goal": stage["goal"], "completion_criteria": stage["completion_criteria"],
+ "method": stage["method"],
+ "constraints": [r["requirement"] for r in memory.requirements if r.get("kind") == "constraint"],
+ "inputs": stage["inputs"], "guidance": stage["guidance"],
+ "target_application": stage.get("target_application"),
+ "facts": [_fact_view(memory.facts[k]) for k in stage["input_fact_keys"] if k in memory.facts],
+ "delivery_state": item["delivery_state"], "remaining": item["remaining"],
+ "failures": [f for f in memory.failures if f["work_item_id"] == item["id"]][-4:],
+ "execution_constraints": [c for c in memory.execution_constraints
+ if c["work_item_id"] == item["id"]][-6:],
+ "actions_remaining": stage["max_actions"] - stage["actions"],
+ "control": "Choose each next action from the NEW observation. DONE requests stage verification, "
+ "not whole-task completion. Do not repeat satisfied input or toggle states."}
diff --git a/src/tandem/memory/state.py b/src/tandem/memory/state.py
new file mode 100644
index 0000000..a5734a8
--- /dev/null
+++ b/src/tandem/memory/state.py
@@ -0,0 +1,229 @@
+"""Host-owned task memory. Plans request work; stage verification commits progress."""
+
+import json
+from copy import deepcopy
+
+LEVELS = {"none": 0, "implemented": 1, "saved": 2, "verified": 3}
+
+
+class TaskMemory:
+ def __init__(self, instruction, requirements):
+ self.instruction = instruction
+ self.requirements = deepcopy(requirements)
+ self.facts = {}
+ self.materials = {}
+ self.work_items = {}
+ self.artifacts = {}
+ self.failures = []
+ self.execution_constraints = []
+ self.active_stage = None
+ self.stage_count = 0
+ self.revision = 0
+ self.known_evidence = set()
+ self.last_rejections = []
+ sources = {f["key"]: f for r in self.requirements for f in r["facts_needed"]}
+ self.source_keys = set(sources)
+ self.stats = {"stages": 0, "verified_stages": 0, "gui_actions": 0,
+ "cli_actions": 0, "new_facts": 0, "delivery_advances": 0,
+ "source_parts": 0, "stages_without_progress": 0}
+ for req in self.requirements:
+ req.update(status="unknown", evidence_refs=[])
+ if req.get("kind") == "constraint":
+ continue
+ needed = [sources[key] for key in dict.fromkeys(
+ key for keys in req["unit_fact_keys"] for key in keys)]
+ preparation = f"{req['id']}:prepare"
+ if needed:
+ self._add(preparation, req, "prepare",
+ "Acquire missing source facts for a ready unit", [], needed)
+ for index, unit in enumerate(req["units"], 1):
+ keys = req["unit_fact_keys"][index - 1]
+ self._add(f"{req['id']}:unit:{index}", req, "deliver", unit, [],
+ [sources[key] for key in keys])
+
+ def _add(self, key, req, kind, goal, dependencies, facts):
+ self.work_items[key] = {
+ "id": key, "requirement_ids": [req["id"]], "kind": kind, "goal": goal,
+ "completion_criteria": goal, "depends_on": dependencies,
+ "facts_needed": facts, "status": "pending",
+ "delivery_state": "none", "remaining": [], "evidence_refs": [],
+ }
+
+ def _facts_ready(self, item):
+ return all(self.facts.get(f["key"], {}).get("status") == "confirmed"
+ and self.facts[f["key"]]["scope"] == "task"
+ and self.facts[f["key"]].get("complete", True) for f in item["facts_needed"])
+
+ def preparation_target(self, item):
+ units = [item] if item["kind"] == "deliver" else [
+ w for w in self.work_items.values() if w["kind"] == "deliver"
+ and w["status"] != "done" and w["requirement_ids"] == item["requirement_ids"]]
+ choices = [(unit, [f for f in unit["facts_needed"]
+ if not self._facts_ready({"facts_needed": [f]})]) for unit in units]
+ choices = [(unit, missing) for unit, missing in choices if missing]
+ return min(choices, key=lambda pair: len(pair[1])) if choices else (item, [])
+
+ def ready_items(self):
+ for item in self.work_items.values():
+ if item["kind"] == "prepare" and item["status"] != "done" and self._facts_ready(item):
+ item.update(status="done", evidence_refs=[ref for f in item["facts_needed"]
+ for ref in self.facts[f["key"]]["evidence_refs"]])
+ return sorted([i for i in self.work_items.values() if i["status"] != "done"
+ and all(self.work_items[d]["status"] == "done" for d in i["depends_on"])],
+ key=lambda i: (i["kind"] != "deliver", i.get("stalled_stages", 0),
+ len(self.preparation_target(i)[1])))
+
+ def start(self, plan, step):
+ item = self.work_items[plan["work_item_id"]]
+ if item not in self.ready_items():
+ raise ValueError("Stage work item is not ready or has already completed")
+ self.stage_count += 1
+ self.active_stage = {**deepcopy(plan), "id": f"S{self.stage_count}",
+ "started_step": step, "actions": 0, "status": "running"}
+ if plan["purpose"] == "prepare":
+ unit, missing = self.preparation_target(item)
+ self.active_stage.update(prepares_unit=unit["id"], fact_targets=deepcopy(missing))
+ item["status"] = "active"
+ self.stats["stages"] += 1
+ return self.active_stage
+
+ def record_action(self, record):
+ self.active_stage["actions"] += 1
+ self.stats["cli_actions" if record["kind"] == "cli" else "gui_actions"] += 1
+ if record["returncode"] != 0:
+ error = (record.get("stderr") or record.get("stdout") or "Execution failed")[-700:]
+ self._failure(error, [record["evidence_ref"]])
+
+ def _failure(self, error, refs):
+ stage = self.active_stage
+ matches = [f for f in self.failures if f["work_item_id"] == stage["work_item_id"]
+ and f["error"] == error]
+ if matches:
+ matches[-1]["count"] += 1
+ matches[-1]["evidence_refs"] = refs
+ else:
+ self.failures.append({"stage_id": stage["id"], "work_item_id": stage["work_item_id"],
+ "method": stage["method"], "error": error,
+ "count": 1, "evidence_refs": refs})
+
+ def apply(self, result, evidence):
+ """Only the fixed active unit is writable; references must belong to real evidence."""
+ stage = self.active_stage
+ item = self.work_items[stage["work_item_id"]]
+ current = set(evidence)
+ self.known_evidence.update(current)
+ allowed = self.known_evidence
+ self.last_rejections = []
+ advanced = False
+ for fact in result["facts"]:
+ fact = deepcopy(fact)
+ refs = fact["evidence_refs"]
+ if not refs or not set(refs) <= allowed:
+ self.last_rejections.append({"key": fact["key"], "reason": "unregistered_evidence"})
+ continue
+ if fact["key"] in self.source_keys:
+ fact["scope"] = "task"
+ old = self.facts.get(fact["key"])
+ entries = fact.get("entries", {})
+ old_entries = old.get("entries", {}) if old else {}
+ conflicts = any(key in old_entries and old_entries[key] != value
+ for key, value in entries.items())
+ extending = bool(old_entries and entries and not conflicts)
+ correction = bool(fact.get("correction_reason") and set(refs) & current)
+ # Incomplete prose sources may arrive a section at a time without structured entries.
+ # Append evidence, never replace the previously observed section or unlock a partial fact.
+ if (old and old["status"] == fact["status"] == "confirmed"
+ and old["scope"] == fact["scope"] == "task" and not old.get("complete", True)
+ and not entries and not old_entries and set(refs) & current and not correction):
+ sections = deepcopy(old.get("observations", [{k: old[k] for k in
+ ("value", "source_ref", "evidence_refs")}]))
+ if fact["value"] not in [section["value"] for section in sections]:
+ sections.append({k: fact[k] for k in ("value", "source_ref", "evidence_refs")})
+ fact["observations"] = sections
+ fact["value"] = "\n".join(section["value"] for section in sections)
+ fact["evidence_refs"] = list(dict.fromkeys(old["evidence_refs"] + refs))
+ extending = True
+ if entries:
+ fact["entries"] = {**old_entries, **entries}
+ fact["value"] = json.dumps(fact["entries"], ensure_ascii=False)
+ fact["evidence_refs"] = list(dict.fromkeys((old or {}).get("evidence_refs", []) + refs))
+ fact["complete"] = bool(fact.get("complete", False) or (old or {}).get("complete", False))
+ if (old and old["status"] == "confirmed" and old["scope"] == "task"
+ and (fact["status"] != "confirmed" or fact["scope"] != "task"
+ or old.get("complete", True) and not fact.get("complete", True)
+ or old["value"] != fact["value"] and not (correction or extending)
+ or conflicts and not correction)):
+ self.last_rejections.append({"key": fact["key"], "reason": "confirmed_source_preserved"})
+ continue
+ if (fact["status"] == "confirmed" and fact["scope"] == "task"
+ and (old is None or old["value"] != fact["value"] or old["status"] != fact["status"]
+ or old.get("complete", True) != fact.get("complete", True))):
+ advanced = advanced or fact["key"] in self.source_keys
+ self.stats["new_facts"] += 1
+ self.facts[fact["key"]] = {**fact, "requirement_ids": item["requirement_ids"],
+ "observed_stage": stage["id"]}
+ for material in result["materials"]:
+ if material["evidence_refs"] and set(material["evidence_refs"]) <= allowed:
+ old = self.materials.get(material["source_ref"], {})
+ new_parts = set(material["covered_parts"]) - set(old.get("covered_parts", []))
+ if new_parts:
+ self.stats["source_parts"] += len(new_parts)
+ summary = old.get("summary", "")
+ if material["summary"] not in summary:
+ summary = (summary + "\n" + material["summary"]).strip()
+ self.materials[material["source_ref"]] = {
+ **material, "summary": summary,
+ "covered_parts": list(dict.fromkeys(old.get("covered_parts", [])
+ + material["covered_parts"])),
+ "evidence_refs": list(dict.fromkeys(old.get("evidence_refs", [])
+ + material["evidence_refs"]))}
+ refs = result["evidence_refs"]
+ grounded = bool(set(refs) & current) and set(refs) <= allowed
+ level = result["delivery_state"]
+ delivering = item["kind"] == "deliver" and stage["purpose"] != "prepare"
+ if (grounded and delivering and LEVELS[level] > LEVELS[item["delivery_state"]]
+ and (level != "verified" or result["stage_outcome"] == "met")):
+ item["delivery_state"] = level
+ advanced = True
+ self.stats["delivery_advances"] += 1
+ item.update(status="pending", remaining=result["remaining"], evidence_refs=refs if grounded else [])
+ preparation_met = item["kind"] == "prepare" and self._facts_ready(item)
+ if grounded and result["stage_outcome"] == "met":
+ self.stats["verified_stages"] += 1
+ if preparation_met or (delivering and result["unit_met"]
+ and level == "verified"):
+ item["status"] = "done"
+ advanced = True
+ for check in result["requirement_checks"]:
+ if check["evidence_refs"] and set(check["evidence_refs"]) <= allowed:
+ req = next((r for r in self.requirements if r["id"] == check["id"]), None)
+ if req is not None:
+ req.update(status=check["status"], evidence_refs=check["evidence_refs"])
+ for artifact in result["artifacts"]:
+ if not artifact["evidence_refs"] or not set(artifact["evidence_refs"]) <= allowed:
+ continue
+ old = self.artifacts.get(artifact["target_path"], {})
+ self.artifacts[artifact["target_path"]] = {**artifact, "revision": old.get("revision", 0) + 1}
+ for key in result["invalidated_work_item_ids"]:
+ if key in self.work_items and self.work_items[key]["kind"] == "deliver" and grounded:
+ self.work_items[key].update(status="pending", delivery_state="none",
+ remaining=["Contradictory evidence requires reinspection"])
+ if result["stage_outcome"] != "met" and grounded:
+ self._failure("; ".join(result["remaining"]) or result["summary"], refs)
+ self.stats["stages_without_progress"] = 0 if advanced else self.stats["stages_without_progress"] + 1
+ item["stalled_stages"] = 0 if advanced else item.get("stalled_stages", 0) + 1
+ item["last_attempt"] = {k: stage[k] for k in ("route", "purpose", "method", "command")}
+ self.revision += 1
+ self.active_stage = None
+ self.ready_items()
+ return advanced
+
+ def snapshot(self):
+ return deepcopy({"schema_version": 2, "revision": self.revision, "instruction": self.instruction,
+ "requirements": self.requirements, "materials": list(self.materials.values()),
+ "facts": list(self.facts.values()), "work_items": list(self.work_items.values()),
+ "artifacts": list(self.artifacts.values()), "failures": self.failures,
+ "execution_constraints": self.execution_constraints,
+ "memory_update_rejections": self.last_rejections,
+ "active_stage": self.active_stage, "stats": self.stats,
+ "evidence_refs": sorted(self.known_evidence)})
diff --git a/src/tandem/models/jev.py b/src/tandem/models/jev.py
index 18950c9..342b2b8 100644
--- a/src/tandem/models/jev.py
+++ b/src/tandem/models/jev.py
@@ -21,11 +21,12 @@
class JevOmniPolicy:
def __init__(self, endpoint: str, timeout_seconds: float = 300, opener=None,
- image_stage: tuple[str, str] | None = None, endpoint_provider=None):
+ image_stage: tuple[str, str] | None = None, endpoint_provider=None, joint_actions=False):
self.endpoint = str(endpoint).rstrip("/")
self.timeout_seconds = timeout_seconds
self.opener = opener or urllib.request.build_opener(urllib.request.ProxyHandler({}))
self.endpoint_provider = endpoint_provider
+ self.joint_actions = joint_actions
# (写视角目录, 排序服务宿主机读视角目录):/rank 要求 image_path 在服务端可读,
# 截图本体在 adpc 时先暂存到 evobox 中转目录(按内容哈希命名去重)。
self.image_stage = image_stage
@@ -164,6 +165,18 @@ def predict(choice_request: dict[str, Any]) -> tuple[list[float], bool]:
candidate for candidate in candidates if candidate.operation == selected_operation
]
action_request = action_requests[selected_operation]
+ if self.joint_actions:
+ # Keep the operation ranking diagnostic; do not prune other concrete goal actions.
+ descriptions = {choice_id: option for request in action_requests.values()
+ for choice_id, option in zip(
+ request["option_ids"], request["options"], strict=True)}
+ selected_candidates = candidates
+ action_request = {
+ "state": operation_request["state"],
+ "question": "Which complete concrete action should be performed next across ALL operations?",
+ "option_ids": [c.id for c in candidates],
+ "options": [f"{c.operation}: {descriptions[c.id]}" for c in candidates],
+ }
if len(selected_candidates) == 1:
action_scores, action_used_image = [1.0], operation_used_image
else:
diff --git a/src/tandem/models/slow.py b/src/tandem/models/slow.py
index 35dcaf9..aab11a3 100644
--- a/src/tandem/models/slow.py
+++ b/src/tandem/models/slow.py
@@ -5,18 +5,78 @@
from __future__ import annotations
+import base64
+import io
import json
import time
import urllib.error
import urllib.request
from pathlib import Path
+from PIL import Image
+
from .key_pool import ApiKeyPool, PoolExhausted, classify_api_error, retry_after_seconds
# 空正文(全留在思考通道)或截断时,下一次重试升档思考强度,避免原样重发重复失败
_REASON_EFFORT_LADDER = {"low": "medium", "medium": "high"}
+def _pack_images(payload, quality, max_bytes):
+ """Pack full-resolution send copies; only exactly identical decoded frames share an image.
+
+ Mutate the existing messages list so validators can still append correction turns on retry.
+ Never write source files or discard distinct evidence frames to meet the body budget.
+ """
+ frames, pixels = [], []
+ stats = {"image_count": 0, "duplicate_images": 0, "source_image_bytes": 0}
+ for message in payload["messages"]:
+ if not isinstance(message.get("content"), list):
+ continue
+ for part in message["content"]:
+ if part.get("type") != "image_url":
+ continue
+ url = part["image_url"]["url"]
+ if not url.startswith("data:image/"):
+ continue
+ raw = base64.b64decode(url.split(",", 1)[1])
+ with Image.open(io.BytesIO(raw)) as source:
+ frame = source.convert("RGB")
+ identity = (frame.size, frame.tobytes())
+ stats["image_count"] += 1
+ stats["source_image_bytes"] += len(raw)
+ if identity in pixels:
+ ordinal = pixels.index(identity) + 1
+ part.clear()
+ part.update(type="text", text=f"Same screenshot as supplied image #{ordinal} (exact pixels).")
+ stats["duplicate_images"] += 1
+ else:
+ pixels.append(identity)
+ frames.append((part, frame))
+ while frames:
+ image_bytes = 0
+ for part, frame in frames:
+ encoded = io.BytesIO()
+ frame.save(encoded, format="JPEG", quality=quality, subsampling=0, optimize=True)
+ raw = encoded.getvalue()
+ image_bytes += len(raw)
+ part["image_url"]["url"] = "data:image/jpeg;base64," + base64.b64encode(raw).decode("ascii")
+ stats.update(image_quality=quality, sent_image_bytes=image_bytes)
+ if len(json.dumps(payload).encode("utf-8")) <= max_bytes or quality == 65:
+ break
+ quality = max(65, quality - 10)
+ while frames and len(json.dumps(payload).encode("utf-8")) > max_bytes and max(
+ max(frame.size) for _, frame in frames
+ ) > 960:
+ for part, frame in frames:
+ frame.thumbnail((max(960, int(frame.width * 0.8)), max(540, int(frame.height * 0.8))))
+ encoded = io.BytesIO()
+ frame.save(encoded, format="JPEG", quality=quality, subsampling=0, optimize=True)
+ part["image_url"]["url"] = "data:image/jpeg;base64," + base64.b64encode(
+ encoded.getvalue()).decode("ascii")
+ return {**stats, "unique_images": len(frames), "sent_image_sizes": [list(f.size) for _, f in frames]}
+
+
+
def extract_json_object(text: str):
"""从模型正文中抢救 JSON:剥围栏后,若剩余文本不是纯 JSON,抽取首个平衡的 {…}。
返回解析后的 dict;无 JSON 或解析失败返回 None。"""
@@ -84,13 +144,19 @@ def json_request(self, messages, validate, *, role):
"model": credentials.get("model") or "unknown",
"messages": messages,
"stream": False,
- "max_tokens": self.config.slow_max_tokens,
+ "max_tokens": (min(self.config.slow_max_tokens, self.config.slow_verify_max_tokens)
+ if role == "verify_stage" else self.config.slow_max_tokens),
}
effort = self.config.slow_reasoning_effort
+ if role == "verify_stage" and self.config.slow_verify_reasoning_effort is not None:
+ effort = self.config.slow_verify_reasoning_effort
if effort == "off":
payload["enable_thinking"] = False
elif effort is not None:
payload["reasoning_effort"] = effort
+ image_stats = (_pack_images(payload, self.config.slow_image_quality,
+ self.config.slow_max_request_bytes)
+ if self.config.execution_mode == "goal" else {})
def build_request():
return urllib.request.Request(
@@ -125,6 +191,7 @@ def audit(event, started, attempt, **details):
"time": time.time(),
"elapsed_seconds": round(time.perf_counter() - started, 3),
"request_bytes": len(request.data),
+ **image_stats,
**details,
}
text = json.dumps(self.key_pool.redact(record), ensure_ascii=False)
diff --git a/src/tandem/planning/goal_planner.py b/src/tandem/planning/goal_planner.py
new file mode 100644
index 0000000..805145e
--- /dev/null
+++ b/src/tandem/planning/goal_planner.py
@@ -0,0 +1,258 @@
+"""One self-contained subgoal, not a speculative list of future UI actions."""
+
+import base64
+import json
+
+from ..memory.context import _fact_view, planning_context
+
+REQUIREMENTS = """Extract the ORIGINAL task into JSON {requirements:[{id,requirement,output_path,
+kind,units,unit_fact_keys,facts_needed}]}. kind is delivery or constraint. units is a list of STRINGS with
+delivery units with their completion criteria; enumerate explicitly requested pages/items, otherwise
+use one unit. Constraints have units:[]. facts_needed is [{key,description}] naming only essential
+stable source information acquired from supplied materials; reuse the SAME key across requirements.
+Never register current focus, open windows or selection as required source facts. Split large source
+packets into the concrete fields or small sections needed by individual deliverables. unit_fact_keys
+is an array parallel to units: each entry lists ONLY the facts_needed keys required by THAT unit.
+Keys may reference definitions in ANY requirement; define each shared source key once. Use [] for a
+unit needing no source reading. Constraints MUST have units:[] AND unit_fact_keys:[], not [[]].
+Include a constraint's required source key on the delivery units that must satisfy that constraint.
+A source-reading dependency must not gate unrelated work.
+Do not list facts already stated in the instruction as facts_needed. Names, titles, URLs and target
+systems explicitly provided by the user are inputs, not missing source facts.
+Keep all original quantities, locations, prohibitions, interaction requirements and saving conditions.
+Enumerate the individual pages/slides/form sections as small delivery units, including when their count
+is explicit but their names are not. Do not collapse 'all 26 slides' into one giant unit. Each unit
+lists only its own missing source content; whole-document consistency remains a final constraint.
+output_path is null unless literally given as an absolute output path in the instruction.
+Do not prescribe actions, guess source facts, inspect evaluators, or introduce hidden grading criteria.
+Return a compact contract, no reasoning, no speculative extra work."""
+
+REQUIREMENTS += '''
+Every requirement MUST contain ALL seven keys, including a plain-string "requirement" describing the
+actual obligation (also for constraints). units MUST be an array of STRINGS, never objects.
+Exact shape example: {"requirements":[{"id":"R1","requirement":"Create and save the requested document",
+"output_path":null,"kind":"delivery","units":["Requested document is complete and saved"],
+"unit_fact_keys":[[]],
+"facts_needed":[]}]}. This example illustrates the format only; extract the real task's obligations.
+'''
+
+PLAN = """Delegate ONE meaningful bounded subtask using the host's task memory. You are the planner;
+the fast GUI choice model will autonomously observe and act, not follow an action list. Return JSON:
+{done:false,work_item_id:"exact ready id",route:"gui|cli",purpose:"prepare|produce|save|verify",
+goal:"specific result",completion_criteria:"stage exit condition",method:"concise approach",
+input_fact_keys:["known key"],inputs:[{label:"field meaning",target_hint:"observed field label",
+text:"exact supplied text",
+input_mode:"replace|insert"}],guidance:"brief necessary navigation/context, not an ordered click script",
+max_actions:24,command:"CLI only; otherwise empty string"}.
+Use a ready work item's FIXED goal and original requirements. Do not rename units or lower final criteria.
+Missing source facts are acquisition targets, NOT a hard gate on starting a delivery unit. First make
+and save the useful portion supported by known facts; leave unknown values unresolved, never guess them.
+Choose a bounded incremental stage exit condition, not the whole unit's acceptance criteria. A partial
+stage may succeed while the unit remains incomplete. Read a missing source only before the next action
+actually needs it, then resume the preserved artifact. Respect explicit user-required reading order.
+input_fact_keys may name ONLY confirmed task-scope keys present in memory.known_facts, NOT keys that
+are merely requested in facts_needed. Use [] before any source facts have been observed.
+For prepare, acquire only missing essential facts; do not reread facts already in memory. Source notes
+are historical evidence, not guaranteed current UI. Reuse confirmed task facts. Temporary inventory,
+selection and page state must be reobserved. Do not fabricate a URL or missing personal information.
+Bound visual source reading to one small section at a time (normally 4-6 observations); the stage-end
+verifier will record its contents. Do not delegate reading several documents and navigating a new site
+as a single preparation goal. Later preparation continues from materials.covered_parts, not page one.
+For GUI give a whole local goal (a form section, adding an attachment, a settings operation), normally
+8-24 possible actions, not merely 'open menu'. It may span pages/dialogs; new targets must be observed,
+not guessed in advance. Provide ALL exact text values it will need from the instruction or known facts.
+For every text input, target_hint must identify its intended observed field (copy the visible label
+when available). For browser navigation use the observed address-bar label; for a document body or
+desktop launcher identify that specific editor/search field. Text is never interchangeable among fields.
+The choice model cannot generate free text. It can click, type provided inputs, use keyboard, scroll,
+wait, request stage verification, or return control. Missing data/new reasoning requires another stage.
+For CLI provide one bounded Bash command, current inputs and expected real results. Prefer actually
+producing/saving deliverables over repeatedly reading tutorials, building audits or protecting unchanged
+files. Inspect edited documents in the actual application before claiming them verified. Keep source
+and final paths distinct. CLI is forbidden if allow_cli=false. Task constraints override routing choices.
+Do not inspect evaluator code, ground truth, rewards, hidden state, unrelated secrets or scoring APIs.
+Previous methods/errors are in memory; changing their description does not make them new. If stalled,
+choose another method or another independent ready unit. Avoid optional research and cosmetic detours.
+Only return {done:true,task_status:"completed|infeasible",current_goal:"evidence-based reason"} for an
+actual whole-task terminal claim. Stage completion is NOT task completion. A blocked unit is not proof
+the task is infeasible. Keep the response short: goal, inputs and one bounded command, no long analysis."""
+
+PLAN += """
+Prefer a ready delivery unit over more preparation. Preparation is bounded acquisition for ONE missing
+field or source section, not an all-document audit or a pixel-analysis research project. State the exact
+observable subgoal, never 'Acquire the required source facts'. Reuse saved extractions and artifacts.
+After two stages without new required facts or delivery progress, use a genuinely different method or
+another ready unit. Do not add optional diagnostic outputs as new completion requirements. Make and
+save a usable increment as soon as its inputs are known; preserve all original final requirements.
+historical_ui_notes are prior observations, NOT the current interface. Current screenshot/controls win.
+For prepare, follow next_acquisition: acquire the stated missing registered fields for that ONE unit.
+Do not inventory every dropdown or inspect source code before filling fields whose values are known.
+Use exact registered keys for acquired contents; fact_index lists already observed auxiliary sources,
+not new dependencies. Do not reread auxiliary sources solely because their full text is omitted here.
+Never type into an unrelated/Untitled document. Confirm the target application and document first.
+For editing an existing document, include target_document:"exact filename" on GUI produce/save stages.
+If it is not currently open in the foreground, first use a prepare stage to open/switch to it. A
+navigation-only prepare stage may have no source fact targets. For websites or creating a NEW document,
+target_document may be null. Available GUI operations include right-click, double-click, Escape,
+Alt+Tab, Ctrl+O, keyboard navigation, supplied text, scrolling and screenshot-grounded targets.
+known_facts includes partial and auxiliary source content; complete:false is usable for a partial
+increment but does NOT mean the source or unit is complete. Never guess ownership/identity when joining
+records from different sources. Do not keep preparing solely to make every facts_needed entry complete.
+If an observed fact is indexed but its text is omitted/excerpted, return ONLY {read_memory:["key",...]}
+for up to six keys, at most twice before planning. This reads stored evidence without reopening files.
+Failure constraints, remaining work and rejected memory updates are actionable state, not suggestions
+to repeat the same attempt with different wording. When the executor reports a blocked cycle, choose
+another interaction, another source or an independent ready unit.
+For a GUI stage that needs to open or switch applications, include target_application:"exact app name".
+This explicit input will also be available for typing in the desktop launcher; do not leave it only in
+prose guidance. Set it to null when not applicable. Prefer CLI to open an installed application or a
+known document when allowed, then delegate actual UI editing to JEV. Do not alternate the application
+drawer open/closed or repeat a covered target. Resolve the visible overlay or use another interaction.
+"""
+
+
+def goal_requirements(client, instruction):
+ messages = [{"role": "system", "content": REQUIREMENTS}, {"role": "user", "content": instruction}]
+
+ def valid(v):
+ okay = (isinstance(v.get("requirements"), list) and bool(v["requirements"])
+ and all(isinstance(r, dict) and isinstance(r.get("id"), str)
+ and isinstance(r.get("requirement"), str)
+ and r.get("kind") in {"delivery", "constraint"}
+ and isinstance(r.get("units"), list)
+ and (r.get("kind") == "constraint" or bool(r["units"]))
+ and all(isinstance(u, str) for u in r["units"])
+ and isinstance(r.get("unit_fact_keys"), list)
+ and len(r["unit_fact_keys"]) == len(r["units"])
+ and all(isinstance(keys, list) and all(isinstance(k, str) for k in keys)
+ for keys in r["unit_fact_keys"])
+ and isinstance(r.get("facts_needed"), list)
+ and all(isinstance(f, dict) and isinstance(f.get("key"), str)
+ and isinstance(f.get("description"), str) for f in r["facts_needed"])
+ for r in v["requirements"]))
+ if okay:
+ registered = {f["key"] for r in v["requirements"] for f in r["facts_needed"]}
+ okay = all(set(keys) <= registered for r in v["requirements"] for keys in r["unit_fact_keys"])
+ if not okay:
+ messages.append({"role": "user", "content":
+ "Invalid requirements JSON. Every entry needs id, requirement (plain text, mandatory), "
+ "output_path, kind, units (array of STRINGS, not objects), unit_fact_keys "
+ "(one array of source key strings per unit) and facts_needed "
+ "(array of {key,description}). Include constraint text in requirement; units:[] is not "
+ "a substitute. Constraints need units:[] AND unit_fact_keys:[], not [[]]. "
+ "Shared source keys may be defined in any requirement. Return the COMPLETE object "
+ "{requirements:[...]}, not an isolated fact or requirement."})
+ return okay
+
+ value = client.json_request(messages, valid, role="task_requirements")
+ for r in value["requirements"]:
+ path = r.get("output_path")
+ if not isinstance(path, str) or not path.startswith("/") or path not in instruction:
+ r["output_path"] = None
+ return value["requirements"]
+
+
+def goal_error(value, memory, allow_cli):
+ """Name the rejected field/constraint so JSON repair can actually change the request."""
+ if type(value.get("done")) is not bool:
+ return "done must be a boolean"
+ if value["done"]:
+ return ("" if value.get("task_status") in ("completed", "infeasible")
+ and isinstance(value.get("current_goal"), str) and value["current_goal"].strip()
+ else "terminal claims require task_status and a nonempty current_goal")
+ ready = {i["id"]: i for i in memory.ready_items()}
+ if not isinstance(value.get("work_item_id"), str):
+ return "work_item_id must be a string naming an exact ready unit"
+ item = ready.get(value.get("work_item_id"))
+ if item is None:
+ return "work_item_id must be one of " + json.dumps(list(ready))
+ if item.get("stalled_stages", 0) >= 2:
+ previous = item.get("last_attempt", {})
+ if (value.get("route") == previous.get("route") == "cli"
+ and isinstance(value.get("command"), str)
+ and value["command"].strip() == previous.get("command", "").strip()):
+ return "command repeats the same CLI attempt after two stalled stages; change method or work unit"
+ if value.get("route") not in (("gui", "cli") if allow_cli else ("gui",)):
+ return "route must be gui" + (" or cli" if allow_cli else "; CLI is disabled")
+ purposes = ("prepare",) if item["kind"] == "prepare" else ("prepare", "produce", "save", "verify")
+ if value.get("purpose") not in purposes:
+ return "purpose must be one of " + json.dumps(sorted(purposes))
+ for key in ("goal", "completion_criteria", "method"):
+ if not isinstance(value.get(key), str) or not value[key].strip():
+ return key + " must be a nonempty string"
+ if not isinstance(value.get("guidance"), str):
+ return "guidance must be a string (empty allowed)"
+ for key in ("target_document", "target_application"):
+ if value.get(key) is not None and (not isinstance(value[key], str) or not value[key].strip()):
+ return key + " must be a nonempty string or null"
+ if type(value.get("max_actions")) is not int or not 1 <= value["max_actions"] <= 24:
+ return "max_actions must be an integer in 1..24"
+ allowed = [k for k, f in memory.facts.items() if f["status"] == "confirmed" and f["scope"] == "task"]
+ if not isinstance(value.get("input_fact_keys"), list) or any(
+ not isinstance(k, str) or k not in allowed for k in value["input_fact_keys"]):
+ return "input_fact_keys must contain only acquired confirmed task keys: " + json.dumps(allowed)
+ if not isinstance(value.get("inputs"), list):
+ return "inputs must be an array (empty allowed)"
+ for index, entry in enumerate(value["inputs"]):
+ if (not isinstance(entry, dict) or not isinstance(entry.get("label"), str)
+ or not isinstance(entry.get("text"), str)
+ or entry.get("input_mode") not in ("replace", "insert")):
+ return f"inputs[{index}] requires label:string, text:string, input_mode:replace|insert"
+ if "target_hint" in entry and (not isinstance(entry["target_hint"], str)
+ or not entry["target_hint"].strip()):
+ return f"inputs[{index}].target_hint must identify the intended field"
+ if not isinstance(value.get("command"), str):
+ return "command must be a string; use an empty string for GUI"
+ if value["route"] == "cli" and not value["command"].strip():
+ return "command must be a nonempty Bash command for CLI"
+ return ""
+
+
+def valid_goal(value, memory, allow_cli):
+ return not goal_error(value, memory, allow_cli)
+
+
+def plan_goal(client, instruction, screenshot, targets, recent, tree, verified_focus=None,
+ requirements=None, remaining_steps=None, *, memory, allow_cli=True, route_policy="auto"):
+ context = {"task": instruction, "memory": planning_context(memory), "allow_cli": allow_cli,
+ "route_policy": route_policy, "remaining_actions": remaining_steps,
+ "visible_controls": [t.compact() for t in targets], "recent": recent[-4:]}
+ messages = [{"role": "system", "content": PLAN}, {"role": "user", "content": [
+ {"type": "text", "text": json.dumps(context, ensure_ascii=False)},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,"
+ + base64.b64encode(screenshot.read_bytes()).decode("ascii")}},
+ ]}]
+ reads = 0
+
+ def validate(v):
+ if set(v) == {"read_memory"}:
+ okay = (reads < 2 and isinstance(v["read_memory"], list) and 1 <= len(v["read_memory"]) <= 6
+ and all(isinstance(k, str) and k in memory.facts and memory.facts[k]["scope"] == "task"
+ for k in v["read_memory"]))
+ error = ("read_memory needs 1..6 existing task-scope keys and at most two reads"
+ if not okay else "")
+ else:
+ error = goal_error(v, memory, allow_cli)
+ if error:
+ messages.append({"role": "user", "content":
+ "Invalid goal-stage JSON: " + error + ". Return the COMPLETE corrected compact stage "
+ "object, preserving evidence and original requirements. Do not replay an executed action."})
+ return not error
+
+ while True:
+ value = client.json_request(messages, validate, role="plan_stage")
+ if set(value) != {"read_memory"}:
+ application = value.get("target_application")
+ if not value["done"] and value["route"] == "gui" and application and not any(
+ i["text"] == application for i in value["inputs"]):
+ value["inputs"].append({"label": "Target application name for desktop search",
+ "target_hint": "Desktop search", "text": application,
+ "input_mode": "replace"})
+ return value
+ reads += 1
+ messages.append({"role": "assistant", "content": json.dumps(value)})
+ messages.append({"role": "user", "content": json.dumps({
+ "stored_source_facts": [_fact_view(memory.facts[k], 16000) for k in value["read_memory"]],
+ "instruction": "Use the stored evidence; now return a bounded stage plan. "
+ "Do not reopen these sources just to recover the same content.",
+ "memory_reads_remaining": 2 - reads}, ensure_ascii=False)})
diff --git a/src/tandem/runtime/goal_stage.py b/src/tandem/runtime/goal_stage.py
new file mode 100644
index 0000000..17247a0
--- /dev/null
+++ b/src/tandem/runtime/goal_stage.py
@@ -0,0 +1,401 @@
+"""Goal-stage loop: receipts/observations each step, one semantic verifier at stage end."""
+
+import json
+import re
+import time
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from urllib.parse import unquote, urlsplit
+
+from PIL import Image, ImageChops, ImageStat
+
+from ..gui.goal_candidates import foreground_tree, goal_candidates
+from ..memory.context import execution_context
+from ..models.ranking import ranking_dict
+from ..protocol.models import CliAction, FailAction
+from .episode import handle_done, resolve_done_probe
+
+
+class ObservationRequired(Exception):
+ """Refresh without executing/replaying an action or consuming an action budget."""
+
+
+def _state_key(targets):
+ return tuple(sorted((t.role, t.name, t.x, t.y, t.value, bool(t.focused), str(t.checked), str(t.selected))
+ for t in targets))
+
+
+def _visual_changed(before, after):
+ frames = []
+ for path in (before, after):
+ with Image.open(path) as image:
+ frames.append(image.crop((0, min(32, image.height // 4), image.width, image.height))
+ .convert("L").resize((160, 90)))
+ return ImageStat.Stat(ImageChops.difference(*frames)).mean[0] > 1.0
+
+
+def _loop_state(targets, tree):
+ root = ET.fromstring(tree or "")
+ return (root.get("foreground_title", ""), root.get("foreground_class", ""),
+ tuple(sorted((t.role, _control_name(t), t.x, t.y, "" if t.role == "slider" else t.value,
+ str(t.checked)) for t in targets)))
+
+
+def _control_name(target):
+ if target.role != "page_tab":
+ return target.name
+ name = re.sub(r"\s+- Memory usage - .+$", "", target.name)
+ return re.sub(r"\s*[–-]\s*(?:Audio playing|Video playing in picture-in-picture mode)$", "", name)
+
+
+def _cycle_actions(recent, state):
+ for period in (2, 3, 4):
+ if len(recent) < period * 2:
+ continue
+ first, second = recent[-period * 2:-period], recent[-period:]
+ if first == second and any(a != first[0] for a in first):
+ return {a["key"] for a in second if a["state"] == state}
+ return set()
+
+
+def _action_key(candidate, targets=()):
+ target = next((t for t in targets if t.id == candidate.target_id), None)
+ name = (f"{target.role}|{_control_name(target)}|{target.x}|{target.y}" if target else
+ candidate.metadata.get("semantic_key") or candidate.description)
+ if candidate.operation == "keypress":
+ name += repr([(t.role, t.name, t.x, t.y) for t in targets if t.focused])
+ parameters = {k: candidate.metadata[k]
+ for k in ("text", "input_mode", "fraction", "key_sequence", "visual_point")
+ if k in candidate.metadata}
+ return (candidate.operation, name, json.dumps(parameters, sort_keys=True, ensure_ascii=False))
+
+
+def _target_effect(old, targets):
+ """Local effect evidence, not semantic stage acceptance. Unknown never means success."""
+ if old["operation"] == "type_text":
+ focus = [t for t in targets if t.focused and t.role != "password_text"]
+ if len(focus) == 1 and focus[0].value:
+ if old.get("field_label") and focus[0].name.casefold() != old["field_label"].casefold():
+ return "not_met"
+ matches = (focus[0].value == old["expected_text"] if old["input_mode"] == "replace"
+ else old["expected_text"] in focus[0].value)
+ return "met" if matches else "not_met"
+ if old["operation"] == "set_slider":
+ target = next((t for t in targets if t.semantic_key == old["target_semantic"]), None)
+ if (target and target.value and target.range_min is not None and target.range_max is not None
+ and target.range_max > target.range_min):
+ fraction = (float(target.value) - target.range_min) / (target.range_max - target.range_min)
+ return "met" if abs(fraction - old["fraction"]) <= 0.04 else "not_met"
+ return "unknown"
+
+
+def _document_mismatch(document, tree, targets):
+ """Reject observed contradictions, not a website's human title or a generic app title."""
+ title = ET.fromstring(tree or "").get("foreground_title", "")
+ filename = Path(document).name.casefold()
+ if Path(document).stem.casefold() in title.casefold():
+ return False
+ addresses = [t.value for t in targets if "address" in t.name.casefold()
+ and "://" in t.value]
+ if Path(document).suffix.casefold() in {".html", ".htm"} and addresses:
+ return all(Path(unquote(urlsplit(value).path)).name.casefold() != filename for value in addresses)
+ return bool(re.search(r"\buntitled\b|\S+\.(?:pptx?|docx?|xlsx?|od[ptsg]|pdf)\b", title, re.I))
+
+
+def _navigation_key(candidate, target, work, state):
+ if candidate.operation == "scroll":
+ return (work, *state[:2], "scroll", "up" if "UP" in candidate.id.upper() else "down")
+ if target and re.search(r"\b(?:zoom|pan)\b", target.name, re.I):
+ return (work, *state[:2], candidate.operation, target.name.casefold())
+ return None
+
+
+class GoalExecution:
+ def __init__(self, memory, max_actions):
+ self.memory = memory
+ self.max_actions = max_actions
+ self.pending = None
+ self.history = []
+ self.end_reason = None
+ self.visual = []
+ self.visual_step = None
+ self.attempts = []
+ self.failed_actions = {}
+ self.navigation_failures = {}
+ self.grounded_states = set()
+ self.no_effect = 0
+ self.empty_stages = 0
+
+ def _emit_state(self, session, step):
+ session.events.append({"route": "task_memory", "step": step,
+ "state": self.memory.snapshot()})
+
+ def _receive(self, session, observation, targets, tree):
+ if self.pending is None:
+ return
+ receipt = observation.last_receipt
+ if receipt is None or not receipt.acknowledged:
+ raise RuntimeError("Missing execution receipt; action is not replayed")
+ old = self.pending
+ self.pending = None
+ signature = _state_key(targets)
+ changed = old["kind"] == "cli" or signature != old["state"] or _visual_changed(
+ old["before_screenshot"], observation.screenshot.path)
+ effect = _target_effect(old, targets)
+ relevant_change = (old["kind"] == "cli" or effect == "met"
+ or effect != "not_met" and old["loop_state"] != _loop_state(targets, tree))
+ if effect == "unknown" and (not targets or old.get("visual_action")):
+ relevant_change = changed
+ if old["kind"] != "cli":
+ key = (self.memory.active_stage["work_item_id"], old["loop_state"], old["candidate_key"])
+ if not relevant_change:
+ self.failed_actions[key] = self.failed_actions.get(key, 0) + 1
+ self.attempts.append({"work": key[0], "state": old["loop_state"], "key": old["candidate_key"]})
+ self.attempts = self.attempts[-16:]
+ self.no_effect = 0 if relevant_change else self.no_effect + 1
+ record = {"step": observation.step_idx, "evidence_ref": f"A{observation.step_idx}",
+ "kind": old["kind"], "operation": old["operation"], "target": old["target"],
+ "action": old["action"], "returncode": receipt.returncode,
+ "stdout": receipt.stdout, "stderr": receipt.stderr,
+ "full_output_ref": receipt.full_output_ref, "screenshot": observation.screenshot.path,
+ "observed_controls": [t.compact() for t in targets][:40],
+ "source_frame": self.memory.active_stage["purpose"] == "prepare",
+ "signature": signature, "observed_change": changed,
+ "relevant_change": relevant_change, "target_effect": effect,
+ "navigation_key": old.get("navigation_key")}
+ self.history.append(record)
+ self.memory.record_action(record)
+ session.recent.append({"operation": old["operation"], "target": old["target"],
+ "goal": self.memory.active_stage["goal"],
+ "outcome": "changed" if relevant_change else "no_effect",
+ "returncode": receipt.returncode,
+ "evidence_ref": record["evidence_ref"]})
+ session.events.append({"route": "goal_action", "step": observation.step_idx,
+ "record": record, "effect_verified": False})
+ if receipt.returncode != 0:
+ self.end_reason = "execution_error"
+ elif old["kind"] == "cli":
+ self.end_reason = "cli_returned"
+ elif self.memory.active_stage["actions"] >= self.memory.active_stage["max_actions"]:
+ self.end_reason = "stage_action_limit"
+ elif (record["navigation_key"] and len(self.history) >= 4
+ and all(h.get("navigation_key") == record["navigation_key"] for h in self.history[-4:])):
+ self.end_reason = "repeated_navigation_checkpoint"
+ elif self.no_effect >= 3 and (session.components.grounder is None
+ or (self.memory.active_stage["work_item_id"], _loop_state(targets, tree))
+ in self.grounded_states):
+ self.end_reason = "repeated_action_without_observable_progress"
+
+ def _finish(self, session, observation):
+ reference = f"{self.memory.active_stage['id']}:O{observation.step_idx}"
+ evidence = {h["evidence_ref"]: h for h in self.history}
+ evidence[reference] = {"screenshot": observation.screenshot.path}
+ started = time.perf_counter()
+ result = session.components.stage_verifier(
+ session.instruction, Path(observation.screenshot.path), memory=self.memory,
+ history=[{k: v for k, v in h.items() if k != "signature"} for h in self.history],
+ end_reason=self.end_reason, evidence=evidence)
+ stage_id = self.memory.active_stage["id"]
+ work = self.memory.active_stage["work_item_id"]
+ source_progress = (self.memory.stats["new_facts"], self.memory.stats["source_parts"])
+ advanced = self.memory.apply(result, evidence)
+ acquired = source_progress != (self.memory.stats["new_facts"], self.memory.stats["source_parts"])
+ if advanced or acquired:
+ self.navigation_failures = {k: v for k, v in self.navigation_failures.items() if k[0] != work}
+ self.memory.execution_constraints = [c for c in self.memory.execution_constraints
+ if not (c["work_item_id"] == work
+ and "navigation_key" in c)]
+ elif self.end_reason == "repeated_navigation_checkpoint":
+ key = self.history[-1]["navigation_key"]
+ self.navigation_failures[key] = self.navigation_failures.get(key, 0) + 1
+ self.empty_stages = self.empty_stages + 1 if not self.history and not advanced and not acquired else 0
+ if self.memory.work_items[work]["status"] == "done":
+ self.failed_actions = {k: v for k, v in self.failed_actions.items() if k[0] != work}
+ self.attempts = [a for a in self.attempts if a["work"] != work]
+ self.memory.execution_constraints = [c for c in self.memory.execution_constraints
+ if c["work_item_id"] != work]
+ for rejected in self.memory.last_rejections:
+ session.events.append({"route": "memory_update_rejected", "stage_id": stage_id, **rejected})
+ session.events.append({"route": "stage_verified", "step": observation.step_idx,
+ "stage_id": stage_id, "end_reason": self.end_reason,
+ "evidence": {k: {"screenshot": v["screenshot"],
+ "full_output_ref": v.get("full_output_ref")}
+ for k, v in evidence.items()},
+ "verification": result, "memory_advanced": advanced,
+ "seconds": time.perf_counter() - started})
+ session.recent.append({"operation": "stage_verified", "stage_id": stage_id,
+ "outcome": result["stage_outcome"], "evidence": result["summary"],
+ "remaining": result["remaining"]})
+ self.history, self.end_reason, self.visual = [], None, []
+ self.visual_step = None
+ self.no_effect = 0
+ self._emit_state(session, observation.step_idx)
+ if self.empty_stages >= 3:
+ session.events.append({"route": "goal_stalled", "step": observation.step_idx,
+ "reason": "three_consecutive_stages_without_an_action"})
+ return FailAction(reason="three_consecutive_stages_without_an_action")
+ raise ObservationRequired("stage_verification_returned")
+
+ def act(self, session, observation):
+ tree = session._read_tree(observation)
+ if tree:
+ tree = foreground_tree(tree)
+ screenshot = Path(observation.screenshot.path)
+ targets = session.components.target_builder.build(
+ tree, f"t{observation.step_idx}", 128, infer_button_labels=True, include_sliders=True,
+ context=" ".join(str(v) for k, v in (self.memory.active_stage or {}).items()
+ if k in {"goal", "completion_criteria", "method", "guidance"}
+ ) or session.instruction, filter_occluded=True,
+ ) if tree else []
+ targets = session.components.semantic_enricher(
+ targets, tree, (observation.screenshot.width, observation.screenshot.height),
+ session._read_browser_snapshot(observation))
+ self._receive(session, observation, targets, tree)
+ if self.end_reason:
+ return self._finish(session, observation)
+ if session.done_probe is not None:
+ result = resolve_done_probe(session, observation, tree)
+ if result is not None:
+ return result
+ raise ObservationRequired("terminal_rejected")
+ if self.memory.active_stage is None:
+ started = time.perf_counter()
+ plan = session.components.planner(
+ session.instruction, screenshot, targets, session.recent, tree or "",
+ requirements=session.requirements, remaining_steps=observation.budget.steps_remaining,
+ memory=self.memory)
+ session.events.append({"route": "planner", "step": observation.step_idx,
+ "mode": "goal", "plan": plan,
+ "seconds": time.perf_counter() - started})
+ if plan["done"]:
+ result = handle_done(session, plan, observation, tree)
+ if result is not None:
+ return result
+ raise ObservationRequired("terminal_rejected")
+ plan["max_actions"] = min(plan["max_actions"], self.max_actions)
+ self.memory.start(plan, observation.step_idx)
+ self._emit_state(session, observation.step_idx)
+ raise ObservationRequired("planner_returned")
+ stage = self.memory.active_stage
+ document = stage.get("target_document")
+ if (stage["route"] == "gui" and stage["purpose"] in {"produce", "save"}
+ and stage["actions"] == 0 and document and _document_mismatch(document, tree, targets)):
+ self.end_reason = f"target_document_mismatch: expected {document!r} in the observed foreground"
+ return self._finish(session, observation)
+ if stage["route"] == "cli":
+ action = CliAction(command=stage["command"], timeout_s=session.config.cli_timeout_seconds,
+ want_observation="screenshot")
+ self.pending = {"kind": "cli", "operation": "cli", "target": stage["method"],
+ "action": action.to_dict(), "state": _state_key(targets),
+ "before_screenshot": observation.screenshot.path}
+ session.events.append({"route": "cli", "step": observation.step_idx,
+ "stage_id": stage["id"], "action": action.to_dict()})
+ return action
+ return self._gui(session, observation, targets, tree, screenshot)
+
+ def _gui(self, session, observation, targets, tree, screenshot):
+ context = execution_context(self.memory)
+ stage = self.memory.active_stage
+ state = _loop_state(targets, tree)
+ work = stage["work_item_id"]
+ blocked_navigation = {k for k, count in self.navigation_failures.items()
+ if k[:3] == (work, *state[:2]) and count >= 2}
+ for key in blocked_navigation:
+ constraint = {"work_item_id": work, "window": list(state[:2]), "navigation_key": list(key),
+ "reason": "Repeated navigation changed pixels but two checkpoints acquired no "
+ "source content or delivery progress. Use another interaction/source."}
+ if constraint not in self.memory.execution_constraints:
+ self.memory.execution_constraints.append(constraint)
+ context["avoid_navigation"] = [list(k) for k in blocked_navigation]
+ recent = [a for a in self.attempts if a["work"] == work][-8:]
+ cycle_keys = _cycle_actions(recent, state)
+ cycle = bool(cycle_keys)
+ blocked = {key[2] for key, count in self.failed_actions.items()
+ if key[:2] == (work, state) and count >= 2}
+ if cycle:
+ blocked.update(cycle_keys)
+ for key in blocked:
+ failure_key = (work, state, key)
+ self.failed_actions[failure_key] = max(2, self.failed_actions.get(failure_key, 0))
+ for key in blocked:
+ constraint = {"work_item_id": work, "window": list(state[:2]), "action_key": list(key),
+ "reason": "Repeated action/cycle in the same observed controls. Try a different "
+ "interaction; unrelated animation or new facts do not clear it."}
+ if constraint not in self.memory.execution_constraints:
+ self.memory.execution_constraints.append(constraint)
+ context["avoid_actions"] = [list(key) for key in blocked]
+ semantic_missing = not any("gnome-shell" not in t.parent_context for t in targets)
+ need_grounding = ((semantic_missing or self.no_effect >= 2 or cycle)
+ and (work, state) not in self.grounded_states)
+ if need_grounding and session.components.grounder is not None:
+ self.grounded_states.add((work, state))
+ self._ground(session, observation, screenshot, context)
+ if self.visual:
+ raise ObservationRequired("local_visual_recovery_returned")
+ # No visual target does not remove valid keyboard recovery such as Escape or Alt+Tab.
+ space = session.components.candidate_builder.build(
+ targets, 96, action_inputs=[], context=" ".join(
+ context[k] for k in ("goal", "completion_criteria", "method", "guidance")))
+ space = goal_candidates(space, targets, tree, context["inputs"],
+ (observation.screenshot.width, observation.screenshot.height))
+ if self.visual_step == observation.step_idx and self.visual:
+ space.target_heads["visual"] = self.visual
+ if (work, state) in self.grounded_states:
+ space.controls.pop("visual_fallback", None)
+ for operation in space.target_heads:
+ space.target_heads[operation] = [c for c in space.target_heads[operation]
+ if _action_key(c, targets) not in blocked and _navigation_key(
+ c, next((t for t in targets if t.id == c.target_id), None), work, state)
+ not in blocked_navigation]
+ space.controls = {k: c for k, c in space.controls.items() if _action_key(c, targets) not in blocked}
+ ranking = session.components.selector.rank(
+ stage_goal=context["goal"], intent=json.dumps(context, ensure_ascii=False),
+ active_window=ET.fromstring(tree).get("foreground_title", "") if tree else "",
+ targets=targets, action_space=space,
+ recent_actions=session.recent[-8:], screenshot=screenshot)
+ winner = ranking.winner
+ operation = winner.operation if winner else "return_to_planner"
+ target = next((t for t in targets if winner and t.id == winner.target_id), None)
+ action = session.components.compile_gui(winner, target) if winner else None
+ session.events.append({"route": "gui", "step": observation.step_idx, "mode": "goal",
+ "stage_id": self.memory.active_stage["id"],
+ "plan": {"stage_goal": context["goal"], "current_goal": context["goal"]},
+ "candidates": [c.compact() for c in space.candidates()],
+ "winner": winner.compact() if winner else None,
+ "avoid_actions": context["avoid_actions"],
+ "ranking": ranking_dict(ranking),
+ "action": action.to_dict() if action else None})
+ if operation == "visual_fallback" and session.components.grounder is not None:
+ if (work, state) not in self.grounded_states:
+ self.grounded_states.add((work, state))
+ self._ground(session, observation, screenshot, context)
+ if self.visual:
+ raise ObservationRequired("visual_grounder_returned")
+ if action is None:
+ self.end_reason = "stage_done_requested" if operation == "done" else "returned_to_planner"
+ return self._finish(session, observation)
+ self.pending = {"kind": action.kind, "operation": operation,
+ "target": winner.description, "action": action.to_dict(),
+ "state": _state_key(targets), "loop_state": state,
+ "candidate_key": _action_key(winner, targets),
+ "before_screenshot": observation.screenshot.path,
+ "expected_text": winner.metadata.get("text"),
+ "input_mode": winner.metadata.get("input_mode"),
+ "target_semantic": target.semantic_key if target else None,
+ "field_label": winner.metadata.get("field_label"),
+ "navigation_key": _navigation_key(winner, target, work, state),
+ "visual_action": operation == "scroll" or winner.source == "visual_grounder"
+ or target is not None and target.role == "canvas",
+ "fraction": winner.metadata.get("fraction")}
+ self.visual, self.visual_step = [], None
+ return action
+
+ def _ground(self, session, observation, screenshot, context):
+ plan = {**context, "current_goal": context["goal"], "stage_goal": context["goal"],
+ "action_proposals": [{"operation": "type_text", **v} for v in context["inputs"]]}
+ self.visual = session.components.grounder(session.instruction, plan, screenshot)
+ self.visual_step = observation.step_idx
+ session.events.append({"route": "recovery", "step": observation.step_idx,
+ "stage_id": self.memory.active_stage["id"],
+ "method": "visual_grounding", "candidate_count": len(self.visual),
+ "avoid_actions": context.get("avoid_actions", [])})
diff --git a/src/tandem/runtime/session.py b/src/tandem/runtime/session.py
index 460337c..31fe6e4 100644
--- a/src/tandem/runtime/session.py
+++ b/src/tandem/runtime/session.py
@@ -18,6 +18,7 @@
from ..models.ranking import ranking_dict
from ..protocol.models import Action, CliAction, FailAction, GuiAction, Observation, WaitAction
from .episode import handle_done, resolve_done_probe, verify_pending
+from .goal_stage import GoalExecution
from .stages import StageExecution, recovery_scopes
from .stopping import NO_ACTION_LIMIT, budget_exhausted
@@ -30,6 +31,7 @@ def __init__(self, components: Components, *, config: SessionConfig | None = Non
self.begin(requirements=[]) # 构造即重置,但不触发 requirements 抽取
def begin(self, instruction: str = "", requirements: list | None = None) -> None:
+ self.goal = None
self.instruction = instruction
# requirements=None(真实 run)时走组件抽取(旧 runtime 同款);
# 显式 [](mock 链路)跳过 LLM 调用
@@ -45,9 +47,14 @@ def begin(self, instruction: str = "", requirements: list | None = None) -> None
self.stage = (
StageExecution(self.config.stage_max_actions) if self.config.execution_mode == "stage" else None
)
+ if self.config.execution_mode == "goal":
+ self.goal = GoalExecution(self.components.memory_factory(instruction, requirements),
+ self.config.stage_max_actions)
# ------------------------------------------------------------------ 主入口
def act(self, observation: Observation) -> Action:
+ if self.goal is not None:
+ return self.goal.act(self, observation)
tree = self._read_tree(observation)
if self.pending is not None:
verify_pending(self, observation, tree)
diff --git a/src/tandem/verification/goal_verifier.py b/src/tandem/verification/goal_verifier.py
new file mode 100644
index 0000000..1306c7e
--- /dev/null
+++ b/src/tandem/verification/goal_verifier.py
@@ -0,0 +1,247 @@
+"""One requirement-bound semantic verification per ended goal stage, never per GUI action."""
+
+import base64
+import json
+from pathlib import Path
+
+PROMPT = """Independently verify the ENDED desktop subtask against its fixed work unit and ORIGINAL
+requirements. Do not judge each intermediate action, demand later pages earlier, or trust the actor's
+printed success. Return only JSON:
+{stage_outcome:"met|not_met|uncertain",unit_met:false,delivery_state:"none|implemented|saved|verified",
+summary:"specific observed result",remaining:["concrete missing part"],evidence_refs:["supplied id"],
+facts:[{key:"registered or concrete reusable key",value:"exact content including numbers/units",
+status:"confirmed|uncertain",scope:"task|stage",source_ref:"actual file/page/location",
+evidence_refs:["supplied id"],complete:true}],
+materials:[{source_ref:"actual file/url",covered_parts:["page/section"],
+summary:"reusable source content",evidence_refs:["supplied id"]}],
+requirement_checks:[{id:"R1",status:"met|not_met|uncertain",evidence_refs:["supplied id"]}],
+artifacts:[{target_path:"actual final path",working_path:"actual working path",save_state:"modified|saved",
+evidence_refs:["supplied id"]}],invalidated_work_item_ids:[]}.
+Use [] when no update is established. Reference only the supplied evidence IDs. Preserve original
+requirements and source facts. Extract essential facts NOW from real observed material/CLI output so
+later stages do not reread it; unobserved contents remain unknown. Preparation uses delivery_state=none.
+Use the exact facts_needed key when observed information answers that key. Keep partial source coverage
+in materials even if not all required facts have been acquired. Opening a reference image, a contact
+sheet, an audit or a script
+that reports failure is not delivery progress. A finished stage need not satisfy a whole requirement.
+unit_met=true ONLY when the full fixed work unit is satisfied; for delivery it requires verified level
+and concrete evidence. Saved requires actual persistence to the target, not a working copy or title.
+File existence/compilation does not prove content or function. Do not invent a demand to repeat every
+successful intermediate action. Mark unmet/uncertain criteria explicitly and preserve actual progress.
+Later contradictions invalidate affected units; do not invalidate old work from an uncertain screenshot
+alone. Do not read evaluators, hidden grading rules, answers or rewards. Be concise and evidence-based."""
+
+PROMPT += '''
+scope=task means reusable SOURCE CONTENT, not whole-task completion. scope=stage is only transient UI
+state. A confirmed paragraph or data field is task-scope even while the rest of its source is unread.
+Not seeing a source again does not contradict its recorded facts. Retain confirmed source values;
+change one only with actual contradictory evidence and a nonempty correction_reason on that fact.
+Use the precise registered source key when it is fully answered; store partial sections as separate
+reusable facts/materials. Do not mark an entire packet confirmed when required fields are missing.
+remaining must describe THIS stage's unmet exit conditions, not a list of every later task obligation.
+Only mark unit_met when the original full unit is satisfied. Historical registered evidence may support
+source facts; stage completion must also cite at least one CURRENT evidence ID.
+All top-level keys are mandatory, including invalidated_work_item_ids:[] when nothing is invalidated.
+Empty-update shape: {"stage_outcome":"uncertain","unit_met":false,"delivery_state":"none",
+"summary":"Explain the observed state","remaining":[],"evidence_refs":[],"facts":[],"materials":[],
+"requirement_checks":[],"artifacts":[],"invalidated_work_item_ids":[]}.
+Use actual observed evidence and updates, not this example's empty values. A stage purpose=prepare is
+source acquisition only even when attached to a delivery unit; it cannot finish that delivery unit.
+For lists/tables/options, include entries:{"stable item/row/position":"observed value"} and complete:false
+until the requested coverage is actually observed. New entries extend the old collection; conflicting
+entries or scalar changes require current evidence and correction_reason. Never label a partial list
+as a complete registered dependency. Temporary UI state is not required source content.
+This request contains selected evidence, not the whole execution log. output_omitted_chars marks an
+explicit excerpt: do not claim to have checked omitted contents or a complete source packet. Extract
+only supported fields; ask for the missing section rather than another full-document audit.
+For prepare, answer fact_targets using their EXACT registered keys and record partial coverage;
+unit_met=false, delivery_state=none, and do not assess unrelated delivery requirements.
+For produce/save/verify, check only the current stage exit condition and its fixed delivery unit.
+Record partial implementation/saving without demanding all future task units. Return only memory deltas,
+not a rewrite of existing facts/materials, and no new optional research or inspection requirements.
+Missing source fields do not invalidate an independently useful, correctly implemented partial stage.
+If that bounded exit condition is met, report stage_outcome=met with unit_met=false and the observed
+implemented/saved level. Keep the full unit incomplete and preserve its missing values for later work.
+Incomplete prose facts may be appended across stages: complete:false preserves earlier sections without
+claiming a complete source. Mark complete:true only when the exact registered field/section is fully answered.
+Never replace a known inventory with the contents of a different document under the same scalar key;
+prefer document/field-specific facts and preserve source_ref. Do not guess identity or ownership.
+'''
+
+
+def result_error(value):
+ """Precise contract failures; never invent evidence or coerce unsupported results."""
+ for key, choices in {"stage_outcome": {"met", "not_met", "uncertain"},
+ "delivery_state": {"none", "implemented", "saved", "verified"}}.items():
+ if value.get(key) not in tuple(choices):
+ return key + " must be one of " + str(sorted(choices))
+ for key, kind in {"unit_met": bool, "summary": str}.items():
+ if type(value.get(key)) is not kind:
+ return key + " must be " + kind.__name__
+ lists = ("remaining", "evidence_refs", "facts", "materials", "requirement_checks", "artifacts",
+ "invalidated_work_item_ids")
+ for key in lists:
+ if not isinstance(value.get(key), list):
+ return key + " must be an array (use [] for no updates)"
+ for key in ("remaining", "evidence_refs", "invalidated_work_item_ids"):
+ if any(not isinstance(x, str) for x in value[key]):
+ return key + " must contain only strings"
+ fields = {"facts": ("key", "value", "source_ref"), "materials": ("source_ref", "summary"),
+ "requirement_checks": ("id",), "artifacts": ("target_path", "working_path")}
+ enums = {"facts": {"status": {"confirmed", "uncertain"}, "scope": {"task", "stage"}},
+ "requirement_checks": {"status": {"met", "not_met", "uncertain"}},
+ "artifacts": {"save_state": {"modified", "saved"}}}
+ for collection, strings in fields.items():
+ for index, record in enumerate(value[collection]):
+ path = f"{collection}[{index}]"
+ if not isinstance(record, dict):
+ return path + " must be an object"
+ for key in strings:
+ if not isinstance(record.get(key), str):
+ return path + "." + key + " must be a string"
+ arrays = ("evidence_refs", "covered_parts") if collection == "materials" else ("evidence_refs",)
+ for key in arrays:
+ if not isinstance(record.get(key), list) or any(not isinstance(x, str) for x in record[key]):
+ return path + "." + key + " must be an array of strings"
+ for key, choices in enums.get(collection, {}).items():
+ if record.get(key) not in tuple(choices):
+ return path + "." + key + " must be one of " + str(sorted(choices))
+ if collection == "facts":
+ if "complete" in record and type(record["complete"]) is not bool:
+ return path + ".complete must be a boolean"
+ if "entries" in record and (not isinstance(record["entries"], dict) or any(
+ not isinstance(k, str) or not isinstance(v, str)
+ for k, v in record["entries"].items())):
+ return path + ".entries must map strings to strings"
+ return ""
+
+
+def valid_result(value):
+ return not result_error(value)
+
+
+def _excerpt(text, limit):
+ if len(text) <= limit:
+ return text
+ return text[:limit // 2] + "\n[... omitted source content; not inspected ...]\n" + text[-limit // 2:]
+
+
+def _actions(history, output_budget):
+ """Keep receipts and semantic deltas, not repeated snapshot IDs/ancestry/whole trees."""
+ actions = []
+ previous = {}
+ output_records = max(1, sum(bool(r.get("full_output_ref") or r.get("stdout") or r.get("stderr"))
+ for r in history))
+ limit = max(1000, output_budget // output_records)
+ for record in history:
+ action = {k: record[k] for k in ("evidence_ref", "kind", "operation", "target", "returncode",
+ "observed_change", "relevant_change", "target_effect",
+ "full_output_ref") if k in record}
+ if record.get("kind") != "cli" and record.get("action"):
+ action["action"] = record["action"]
+ output = dict(record)
+ if record.get("full_output_ref"):
+ output.update(json.loads(Path(record["full_output_ref"]).read_text(encoding="utf-8")))
+ omitted = {}
+ for key in ("stdout", "stderr"):
+ text = output.get(key, "")
+ if text:
+ action[key] = _excerpt(text, limit)
+ if len(text) > limit:
+ omitted[key] = len(text) - limit
+ if omitted:
+ action["output_omitted_chars"] = omitted
+ changes, current = [], {}
+ for control in record.get("observed_controls", []):
+ key = (control.get("role"), control.get("name"), control.get("x"), control.get("y"))
+ value = {k: control[k] for k in ("role", "name", "value", "focused", "checked", "selected")
+ if k in control}
+ current[key] = value
+ if value != previous.get(key):
+ changes.append(value)
+ previous = current
+ if changes:
+ action["control_changes"] = changes
+ actions.append(action)
+ return actions
+
+
+def _failure_result(memory, history, end_reason, evidence):
+ """No semantic success claim from a known error or an entirely ineffective local attempt."""
+ pure_no_effect = (history and all(h.get("kind") == "gui"
+ and not h.get("relevant_change", h.get("observed_change"))
+ for h in history))
+ skip = (end_reason == "execution_error" or not history and end_reason != "stage_done_requested"
+ or pure_no_effect and memory.active_stage["purpose"] not in {"save", "verify"})
+ if not skip:
+ return None
+ return {"stage_outcome": "uncertain", "unit_met": False, "delivery_state": "none",
+ "summary": "No semantic acceptance: " + end_reason,
+ "remaining": ["Inspect any partial effects before recovery; do not repeat the failed method."],
+ "evidence_refs": list(evidence)[-1:], "facts": [], "materials": [],
+ "requirement_checks": [], "artifacts": [], "invalidated_work_item_ids": [],
+ "check_mode": "local_failure"}
+
+
+def verify_goal(client, instruction, screenshot, *, memory, history, end_reason, evidence):
+ local = _failure_result(memory, history, end_reason, evidence)
+ if local is not None:
+ return local
+ stage = memory.active_stage
+ work = memory.work_items[stage["work_item_id"]]
+ targets = stage.get("fact_targets", work["facts_needed"])
+ keys = set(stage["input_fact_keys"]) | {f["key"] for f in targets}
+ budget = client.config.stage_verify_max_chars
+ actions = _actions(history, budget // 2)
+ context = {"task": instruction,
+ "stage": {k: v for k, v in stage.items() if k != "command"},
+ "command_excerpt": _excerpt(stage.get("command", ""), 3000),
+ "fixed_work_item": {**work, "facts_needed": targets},
+ "requirements": [{k: r[k] for k in ("id", "requirement", "kind")}
+ for r in memory.requirements if r["id"] in work["requirement_ids"]
+ or r["kind"] == "constraint"],
+ "known_facts": [f for f in memory.facts.values() if f["key"] in keys],
+ "end_reason": end_reason, "actual_actions": actions, "evidence_ids": list(evidence),
+ "historical_evidence_ids": sorted(memory.known_evidence)}
+ text = json.dumps(context, ensure_ascii=False)
+ if len(text) > budget:
+ # Receipt deltas stay intact. Explicitly excerpt source text, never silently claim full coverage.
+ for index, original in enumerate(context["known_facts"]):
+ fact = {k: v for k, v in original.items() if k != "entries"}
+ fact["value"] = _excerpt(fact["value"], max(1000, budget // (2 * max(1, len(keys)))))
+ if fact["value"] != original["value"]:
+ fact["value_excerpt_only"] = True
+ context["known_facts"][index] = fact
+ text = json.dumps(context, ensure_ascii=False)
+ content = [{"type": "text", "text": text}]
+ source_frames = [h["screenshot"] for h in history if h.get("source_frame")]
+ paths = source_frames[-5:] if stage["purpose"] == "prepare" else (
+ [history[0]["screenshot"]] if history else [])
+ frames = [Path(p) for p in dict.fromkeys(paths + [str(screenshot)])]
+ for path in frames:
+ content.extend([{"type": "text", "text": "Observed screenshot: " + str(path)},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,"
+ + base64.b64encode(path.read_bytes()).decode("ascii")}}])
+ messages = [{"role": "system", "content": PROMPT}, {"role": "user", "content": content}]
+
+ def validate(value):
+ error = result_error(value)
+ if not error:
+ allowed = set(evidence) | memory.known_evidence
+ records = [value, *value["facts"], *value["materials"],
+ *value["artifacts"], *value["requirement_checks"]]
+ unknown = sorted({ref for r in records for ref in r["evidence_refs"] if ref not in allowed})
+ if unknown:
+ error = "evidence_refs contain unregistered IDs: " + json.dumps(unknown)
+ elif value["stage_outcome"] == "met" and not set(value["evidence_refs"]) & set(evidence):
+ error = "a met stage must cite at least one CURRENT evidence ID"
+ if error:
+ messages.append({"role": "user", "content":
+ "Invalid stage verification JSON: " + error + ". Return the COMPLETE corrected object "
+ "with every mandatory key, without changing the observed facts or inventing evidence."})
+ return not error
+
+ result = client.json_request(messages, validate, role="verify_stage")
+ result.update(check_mode="source" if stage["purpose"] == "prepare" else "delivery",
+ context_chars=len(text), image_count=len(frames))
+ return result
diff --git a/src/tandem_eval/__main__.py b/src/tandem_eval/__main__.py
index b4b30a8..8dad50e 100644
--- a/src/tandem_eval/__main__.py
+++ b/src/tandem_eval/__main__.py
@@ -36,7 +36,7 @@ def main(argv: list[str] | None = None) -> int:
run_cmd.add_argument("--allow-cli", choices=["true", "false"], default=None,
help="harness 是否允许 CLI 路由")
run_cmd.add_argument("--route-policy", choices=["auto", "gui", "cli"], default=None)
- run_cmd.add_argument("--execution-mode", choices=["step", "stage"], default=None)
+ run_cmd.add_argument("--execution-mode", choices=["step", "stage", "goal"], default=None)
run_cmd.add_argument("--stage-max-actions", type=int, default=None,
help="阶段内动作数上限,达到后请求剩余工作的重规划")
run_cmd.add_argument("--slow-model", default=None, metavar="NAME",
diff --git a/src/tandem_eval/config_schema.py b/src/tandem_eval/config_schema.py
index e6d9ceb..bb9ae28 100644
--- a/src/tandem_eval/config_schema.py
+++ b/src/tandem_eval/config_schema.py
@@ -241,8 +241,8 @@ def validate(config: dict) -> list[str]:
if not isinstance(execution, dict):
problems.append("harness.execution(应为对象)")
else:
- if execution.get("mode", "step") not in {"step", "stage"}:
- problems.append("harness.execution.mode(应为 step / stage)")
+ if execution.get("mode", "step") not in {"step", "stage", "goal"}:
+ problems.append("harness.execution.mode(应为 step / stage / goal)")
limit = execution.get("stage_max_actions", 24)
if type(limit) is not int or limit < 1:
problems.append("harness.execution.stage_max_actions(应为正整数)")
diff --git a/src/tandem_eval/entry.py b/src/tandem_eval/entry.py
index 7406ec7..df4150f 100644
--- a/src/tandem_eval/entry.py
+++ b/src/tandem_eval/entry.py
@@ -103,8 +103,14 @@ def __init__(self, *, stage_mode=False):
self.stage_mode = stage_mode
def __call__(self, instruction, screenshot, targets, recent, tree,
- verified_focus, requirements, remaining_steps):
+ verified_focus=None, requirements=None, remaining_steps=None, *, memory=None):
self.calls += 1
+ if memory is not None and self.calls == 1:
+ return {"done": False, "work_item_id": memory.ready_items()[0]["id"],
+ "route": "gui", "purpose": "produce", "goal": "mock click",
+ "completion_criteria": "mock receipt", "method": "mock click",
+ "inputs": [], "input_fact_keys": [], "guidance": "", "max_actions": 1,
+ "command": ""}
if self.calls == 1 and targets:
target = targets[0]
plan = {
@@ -146,19 +152,32 @@ def rank(self, **kwargs):
)
-def mock_components(*, stage_mode=False) -> Components:
+def mock_components(*, stage_mode=False, goal_mode=False) -> Components:
from tandem.gui.candidates import ActionCandidateBuilder
from tandem.gui.compile import gui_action
from tandem.gui.proposals import focused_candidates
from tandem.gui.semantics import enrich_targets
from tandem.gui.targets import A11yTargetBuilder
+ from tandem.memory.state import TaskMemory
+
+ def goal_verifier(*args, **kwargs):
+ return {"stage_outcome": "met", "unit_met": True, "delivery_state": "verified",
+ "summary": "mock evidence only", "remaining": [],
+ "evidence_refs": list(kwargs["evidence"]), "facts": [], "materials": [],
+ "requirement_checks": [], "artifacts": [], "invalidated_work_item_ids": []}
+
+ def requirements(instruction):
+ return [{"id": "R1", "requirement": instruction, "kind": "delivery", "units": [instruction],
+ "unit_fact_keys": [[]], "facts_needed": [], "output_path": None}] if goal_mode else []
return Components(
planner=_ScriptedPlanner(stage_mode=stage_mode), verifier=_ScriptedVerifier(),
- requirements=lambda instruction: [], selector=_FirstCandidateSelector(),
+ requirements=requirements, selector=_FirstCandidateSelector(),
target_builder=A11yTargetBuilder(), candidate_builder=ActionCandidateBuilder(),
semantic_enricher=enrich_targets,
focus_candidates=focused_candidates, grounder=None, compile_gui=gui_action,
+ memory_factory=TaskMemory if goal_mode else None,
+ stage_verifier=goal_verifier if goal_mode else None,
)
@@ -166,7 +185,8 @@ def build_session(spec: dict, *, llm_tap: LlmTap | None = None) -> TandemSession
config = SessionConfig(**spec.get("harness_config", {}))
benchmark = spec.get("benchmark")
if benchmark == "mock":
- return TandemSession(mock_components(stage_mode=config.execution_mode == "stage"), config=config)
+ return TandemSession(mock_components(stage_mode=config.execution_mode == "stage",
+ goal_mode=config.execution_mode == "goal"), config=config)
if benchmark in {"osworld1", "osworld2", "osworld21"}:
from tandem.components import build_default
from tandem.models.slow import SlowModelClient
diff --git a/src/tandem_eval/env.py b/src/tandem_eval/env.py
index 38f5c5f..77b2bd3 100644
--- a/src/tandem_eval/env.py
+++ b/src/tandem_eval/env.py
@@ -7,10 +7,13 @@
from __future__ import annotations
import hashlib
+import io
from dataclasses import dataclass, field
from pathlib import Path
from typing import Protocol
+from PIL import Image
+
from tandem.protocol.models import (
AccessibilityTree,
Action,
@@ -30,6 +33,8 @@ def reset(self, task: dict) -> Observation: ...
def execute(self, action: Action, *, budget: Budget, task: dict) -> tuple[Receipt, Observation]: ...
+ def observe(self, task: dict, *, budget: Budget, previous: Observation) -> Observation: ...
+
def fetch_task(self, task_id: str) -> dict: ...
def export_outputs(self, paths: list[str], episode_dir) -> list[dict]: ...
@@ -75,7 +80,9 @@ def fetch_task(self, task_id: str, domain: str = "") -> dict:
def _screenshot_ref(self, task_id: str, step_idx: int) -> tuple[str, str]:
key = f"{task_id}:{step_idx}"
if key not in self.shots:
- self.shots[key] = f"png-{key}".encode()
+ image = io.BytesIO()
+ Image.new("RGB", (1920, 1080), "white").save(image, format="PNG")
+ self.shots[key] = image.getvalue()
digest = hashlib.sha256(self.shots[key]).hexdigest()
if self.base_dir is not None:
target = self.base_dir / "screenshots" / f"step_{step_idx:02d}.png"
@@ -140,6 +147,9 @@ def execute(self, action: Action, *, budget: Budget, task: dict) -> tuple[Receip
self.screenshot_counter += 1
return receipt, self._observation(task, step_idx, budget, receipt)
+ def observe(self, task, *, budget, previous):
+ return self._observation(task, previous.step_idx, budget, previous.last_receipt)
+
def export_outputs(self, paths: list[str], episode_dir) -> list[dict]:
records = []
for index, path in enumerate(paths):
diff --git a/src/tandem_eval/osworld.py b/src/tandem_eval/osworld.py
index f8d3905..19685b8 100644
--- a/src/tandem_eval/osworld.py
+++ b/src/tandem_eval/osworld.py
@@ -47,8 +47,9 @@ def png_size(data: bytes) -> tuple[int, int]:
def gui_code(action: GuiAction) -> str:
params = action.parameters
- if action.action_type in {"click", "double_click"}:
- method = "click" if action.action_type == "click" else "doubleClick"
+ if action.action_type in {"click", "double_click", "right_click"}:
+ method = {"click": "click", "double_click": "doubleClick", "right_click": "rightClick"}[
+ action.action_type]
return f"pyautogui.{method}(x={params['x']}, y={params['y']})"
if action.action_type == "hotkey":
steps = []
@@ -66,7 +67,12 @@ def gui_code(action: GuiAction) -> str:
steps.append("pyautogui.press('f2')")
if params.get("input_mode", "replace") == "replace":
steps.append("pyautogui.hotkey('ctrl', 'a')")
- steps.append(f"pyautogui.write({params['text']!r})")
+ if params["text"].isascii():
+ steps.append(f"pyautogui.write({params['text']!r})")
+ else:
+ steps.append("__import__('subprocess').run(['xclip','-selection','clipboard','-t','UTF8_STRING'],"
+ f"input={params['text']!r}.encode('utf-8'),check=True)")
+ steps.append("pyautogui.hotkey('ctrl','v')")
return "; ".join(steps)
raise ValueError(f"不支持的 GUI 动作类型: {action.action_type}")
@@ -126,6 +132,7 @@ def _bootstrap_runtime(self) -> None:
os.environ["OSWORLD_FILE_BASE_URL"] = str(self.asset_root)
self.env: Any = None
self._step_seq = 0
+ self._frame_seq = 0
self.last_screenshot: bytes = b""
self.last_a11y: str = ""
self.last_browser_snapshot: dict | None = None
@@ -182,13 +189,15 @@ def _observation(self, task: dict, step_idx: int, budget: Budget,
receipt: Receipt | None) -> Observation:
stage = self.run_root / "obs_stage" / str(task["id"])
stage.mkdir(parents=True, exist_ok=True)
- shot_path = stage / f"step_{step_idx:02d}.png"
+ self._frame_seq += 1
+ frame_name = f"step_{step_idx:02d}_obs_{self._frame_seq:04d}"
+ shot_path = stage / f"{frame_name}.png"
shot_path.write_bytes(self.last_screenshot)
a11y_path: str | None = None
digest = node_count = None
present = bool(self.last_a11y.strip())
if present:
- a11y_path = str(stage / f"step_{step_idx:02d}.xml")
+ a11y_path = str(stage / f"{frame_name}.xml")
Path(a11y_path).write_text(self.last_a11y, encoding="utf-8")
digest = hashlib.sha256(self.last_a11y.encode()).hexdigest()
node_count = self.last_a11y.count("<")
@@ -196,7 +205,7 @@ def _observation(self, task: dict, step_idx: int, budget: Budget,
browser_snapshot = None
if self.last_browser_snapshot is not None:
serialized = json.dumps(self.last_browser_snapshot, ensure_ascii=False)
- browser_path = stage / f"step_{step_idx:02d}.dom.json"
+ browser_path = stage / f"{frame_name}.dom.json"
browser_path.write_text(serialized, encoding="utf-8")
browser_snapshot = BrowserSnapshot(
present=True,
@@ -229,6 +238,11 @@ def _capture(self) -> None:
public = {key: value for key, value in diagnostic.items() if key != "url"}
print(json.dumps({"event": "cdp_snapshot", **public}, ensure_ascii=False), flush=True)
+ def observe(self, task, *, budget, previous):
+ """Refresh screenshot, A11y and CDP without executing or replaying any action."""
+ self._capture()
+ return self._observation(task, previous.step_idx, budget, previous.last_receipt)
+
# ------------------------------------------------------------------ 协议
def _seed_setup_cache(self, task_id: str) -> None:
"""把共享盘已缓存的 setup 文件播种进本次 evaluation_cache(旧 runtime.py:532-539),
@@ -291,9 +305,17 @@ def acknowledged(command):
stdout = result.get("output", "")
stderr = result.get("error", "")
returncode = int(result.get("returncode", 0))
+ output_ref = None
+ if kind == "cli":
+ output = self.run_root / "cli_outputs" / f"action_{self._step_seq:04d}.json"
+ output.parent.mkdir(parents=True, exist_ok=True)
+ output.write_text(json.dumps({"stdout": stdout, "stderr": stderr,
+ "returncode": returncode}), encoding="utf-8")
+ output_ref = str(output)
return Receipt(
returncode=returncode, stdout=stdout[-RECEIPT_TAIL:], stderr=stderr[-RECEIPT_TAIL:],
kind=kind if kind in {"cli", "gui", "wait"} else "gui", acknowledged=True,
+ full_output_ref=output_ref,
)
def execute(self, action: Action, *, budget: Budget, task: dict) -> tuple[Receipt, Observation]:
diff --git a/src/tandem_eval/recorder.py b/src/tandem_eval/recorder.py
index fe26c6c..2df4145 100644
--- a/src/tandem_eval/recorder.py
+++ b/src/tandem_eval/recorder.py
@@ -57,6 +57,10 @@ def __init__(self, episode_dir: Path, task: dict, *, a11y_full: bool = False):
self.dir.mkdir(parents=True, exist_ok=True)
self.task = task
self.a11y_full = a11y_full
+ self.executed_steps = 0
+ self.environment_seconds = {}
+ self.model_usage = {}
+ self.fast_model = {"choice_cycles": 0, "seconds": 0.0, "cycles_with_image": 0}
self.started = time.perf_counter()
self._traj = (self.dir / "traj.jsonl").open("w", encoding="utf-8")
self._events = (self.dir / "events.jsonl").open("w", encoding="utf-8")
@@ -65,6 +69,7 @@ def __init__(self, episode_dir: Path, task: dict, *, a11y_full: bool = False):
# ------------------------------------------------------------------ 逐步记录
def traj(self, observation: Observation, action: Action, receipt: Receipt | None) -> None:
+ self.executed_steps += 1
browser_snapshot = observation.browser_snapshot
browser_record = browser_snapshot.to_dict() if browser_snapshot else None
if browser_snapshot and browser_snapshot.present and browser_snapshot.path:
@@ -89,9 +94,40 @@ def traj(self, observation: Observation, action: Action, receipt: Receipt | None
self._traj.flush()
def event(self, event: dict) -> None:
+ if event.get("route") == "environment_timing":
+ phase = event["phase"]
+ self.environment_seconds[phase] = self.environment_seconds.get(phase, 0.0) + event["seconds"]
+ if event.get("route") == "gui" and event.get("ranking"):
+ ranking = event["ranking"]
+ self.fast_model["choice_cycles"] += 1
+ self.fast_model["seconds"] += ranking["latency_ms"] / 1000
+ self.fast_model["cycles_with_image"] += int(ranking["used_image"])
+ if event.get("route") == "stage_verified":
+ event = self._persist_evidence(event)
self._events.write(json.dumps(event, ensure_ascii=False) + "\n")
self._events.flush()
+ def task_memory(self, state: dict) -> None:
+ """Persist the latest host memory and retain its transient evidence before cleanup."""
+ _write_json(self.dir / "task_memory.json", self._persist_evidence(state))
+
+ def _persist_evidence(self, state):
+ def persist(value):
+ if isinstance(value, dict):
+ return {k: persist(v) for k, v in value.items()}
+ if isinstance(value, list):
+ return [persist(v) for v in value]
+ if (isinstance(value, str) and any(part in value for part in ("/obs_stage/", "/cli_outputs/"))
+ and Path(value).is_file()):
+ target = self.dir / "evidence" / Path(value).name
+ target.parent.mkdir(exist_ok=True)
+ if not target.exists():
+ shutil.copy2(value, target)
+ return str(target)
+ return value
+
+ return persist(state)
+
def record_llm_call(self, *, role: str, request: Any, response: Any, **meta: Any) -> None:
"""慢模型客户端的 llm_sink;全量请求/响应落盘,密钥脱敏。"""
self._llm.write(
@@ -110,6 +146,16 @@ def record_llm_call(self, *, role: str, request: Any, response: Any, **meta: Any
+ "\n"
)
self._llm.flush()
+ stats = self.model_usage.setdefault(role, {"calls": 0, "seconds": 0.0, "prompt_tokens": 0,
+ "completion_tokens": 0, "reasoning_tokens": 0, "cached_tokens": 0, "calls_missing_usage": 0})
+ stats["calls"] += 1
+ stats["seconds"] += meta.get("elapsed_seconds", 0.0)
+ usage = response.get("usage") or {}
+ stats["calls_missing_usage"] += int(not usage)
+ for key in ("prompt_tokens", "completion_tokens"):
+ stats[key] += usage.get(key, 0)
+ stats["reasoning_tokens"] += (usage.get("completion_tokens_details") or {}).get("reasoning_tokens", 0)
+ stats["cached_tokens"] += (usage.get("prompt_tokens_details") or {}).get("cached_tokens", 0)
def screenshot(self, step_idx: int, data: bytes) -> None:
target = self.dir / "screenshots" / f"step_{step_idx:02d}.png"
@@ -150,7 +196,7 @@ def finalize(
"task_id": self.task["id"],
"harness": f"tandem {__version__}",
"stop_reason": stop_reason,
- "steps": steps,
+ "steps": self.executed_steps if exception is not None else steps,
"status": status,
"score": score,
"exception_type": type(exception).__name__ if exception else None,
@@ -163,6 +209,9 @@ def finalize(
self.dir / "timing.json",
{
"wall_seconds": round(time.perf_counter() - self.started, 3),
+ "environment_seconds": self.environment_seconds,
+ "model_usage_by_role": self.model_usage,
+ "fast_model": self.fast_model,
"status": status,
},
)
diff --git a/src/tandem_eval/rjob_spec.py b/src/tandem_eval/rjob_spec.py
index 99250ba..88f4e08 100644
--- a/src/tandem_eval/rjob_spec.py
+++ b/src/tandem_eval/rjob_spec.py
@@ -117,6 +117,11 @@ def _harness_config_section(config: dict, slow_api: dict, fast_api: dict) -> dic
"allow_cli": bool(route.get("allow_cli", True)),
"cli_timeout_seconds": int(route.get("cli_timeout_seconds", 90)),
"slow_max_tokens": int(slow_api.get("max_tokens", 16384)),
+ "slow_verify_max_tokens": int(execution.get("verify_max_tokens", 16384)),
+ "slow_verify_reasoning_effort": execution.get("verify_reasoning_effort"),
+ "stage_verify_max_chars": int(execution.get("verify_max_chars", 48000)),
+ "slow_image_quality": int(execution.get("image_quality", 85)),
+ "slow_max_request_bytes": int(execution.get("max_request_bytes", 2 * 1024 * 1024)),
"slow_timeout_seconds": float(slow_api.get("timeout_seconds", 120)),
"slow_reasoning_effort": slow_api.get("reasoning_effort"),
"slow_direct_api": bool(slow_api.get("direct_api", False)),
@@ -193,7 +198,8 @@ def build_job_spec(*, config: dict, run_id: str, repo_root: Path,
"image_stage": _image_stage_section(config),
"harness": {
"name": "tandem",
- "mode": "stage" if harness_config["execution_mode"] == "stage" else "planner_led",
+ "mode": (harness_config["execution_mode"]
+ if harness_config["execution_mode"] in {"stage", "goal"} else "planner_led"),
"models": {"slow": slow_name, "fast": fast_name},
},
"harness_config": harness_config,
diff --git a/src/tandem_eval/runner.py b/src/tandem_eval/runner.py
index 8a9d18c..b84320c 100644
--- a/src/tandem_eval/runner.py
+++ b/src/tandem_eval/runner.py
@@ -8,6 +8,7 @@
from pathlib import Path
from tandem.protocol.models import Observation
+from tandem.runtime.goal_stage import ObservationRequired
from tandem.runtime.session import TandemSession
from .env import budget_of
@@ -25,7 +26,16 @@ def run_episode(env, session: TandemSession, recorder: EpisodeRecorder, task: di
started = time.perf_counter()
steps_used = 0
stop_reason = "max_steps"
- observation: Observation = env.reset(task)
+
+ def environment_call(phase, function, *args, **kwargs):
+ before = time.perf_counter()
+ try:
+ return function(*args, **kwargs)
+ finally:
+ recorder.event({"route": "environment_timing", "phase": phase,
+ "seconds": time.perf_counter() - before})
+
+ observation: Observation = environment_call("setup", env.reset, task)
outputs: list[dict] = []
events_start = len(session.events) # 本题事件起点:act 后增量落盘(planner/快模型/验证)
@@ -34,6 +44,8 @@ def drain_events() -> None:
for event in session.events[events_start:]:
recorder.event(event)
events_start = len(session.events)
+ if session.goal is not None:
+ recorder.task_memory(session.goal.memory.snapshot())
while True:
budget = budget_of(
@@ -50,19 +62,27 @@ def drain_events() -> None:
stop_reason = "wall_time_limit"
break
observation = replace(observation, budget=budget)
- action = session.act(observation)
- drain_events()
+ try:
+ action = session.act(observation)
+ except ObservationRequired as refresh:
+ recorder.event({"route": "observation_refresh", "step": steps_used,
+ "reason": str(refresh), "action_replayed": False})
+ observation = environment_call("observe", env.observe, task, budget=budget, previous=observation)
+ continue
+ finally:
+ drain_events()
recorder.event({"step": steps_used, "kind": "action", "action": action.to_dict()})
if getattr(action, "kind", "") in TERMINAL_KINDS:
stop_reason = "done" if action.kind == "finish" else action.reason
# 终态仍须送达 VM(DONE/FAIL 触发评分钩子,旧 runtime 同款)
- env.execute(action, budget=budget, task=task)
+ environment_call("execute", env.execute, action, budget=budget, task=task)
if action.kind == "finish":
- outputs = env.export_outputs(
+ outputs = environment_call(
+ "export", env.export_outputs,
list(action.outputs) + list(extra_exports or []), recorder.dir
)
break
- receipt, observation = env.execute(action, budget=budget, task=task)
+ receipt, observation = environment_call("execute", env.execute, action, budget=budget, task=task)
recorder.traj(observation, action, receipt)
# 环境提供素材属性时写入 episode 目录(真实环境 M3 同样暴露 last_* 属性)
shot = getattr(env, "last_screenshot", None)
@@ -72,7 +92,9 @@ def drain_events() -> None:
if a11y_text:
recorder.a11y(steps_used, a11y_text)
steps_used += 1
- raw = env.score(task)
+ raw = environment_call("evaluate", env.score, task)
+ if session.goal is not None:
+ recorder.task_memory(session.goal.memory.snapshot())
score = official_score(raw) if raw is not None else None
return recorder.finalize(
stop_reason=stop_reason,
@@ -129,6 +151,8 @@ def run(spec: dict, *, env, session: TandemSession, tasks: list[TaskRef],
)
)
finally:
+ if recorder is not None and session.goal is not None:
+ recorder.task_memory(session.goal.memory.snapshot())
if llm_tap is not None:
llm_tap.current = None
# obs_stage 是 episode 内工作区(截图正本已由 recorder 写入 episode 目录,
diff --git a/tests/eval/test_experiment_script.py b/tests/eval/test_experiment_script.py
index 9c2d3ea..9773b32 100644
--- a/tests/eval/test_experiment_script.py
+++ b/tests/eval/test_experiment_script.py
@@ -54,6 +54,16 @@ def test_invalid_experiment_name_is_rejected(tmp_path):
assert not (tmp_path / "bad name").exists()
+def test_dry_run_selects_goal_mode_without_changing_script_default(tmp_path):
+ result = subprocess.run([
+ "bash", str(SCRIPT), "--experiment", "goal-smoke", "--tasks", "008", "--execution-mode", "goal",
+ "--stage-max-actions", "2", "--dry-run"], capture_output=True, text=True,
+ env={**os.environ, "TANDEM_EXPERIMENT_ROOT": str(tmp_path)})
+ assert result.returncode == 0, result.stderr
+ manifest = json.loads((tmp_path / "goal-smoke/manifest.json").read_text())
+ assert manifest["execution_mode"] == "goal" and manifest["stage_max_actions"] == 2
+
+
def test_dry_run_uses_example_when_private_config_is_missing(tmp_path):
env = {
**os.environ,
diff --git a/tests/eval/test_mock_chain.py b/tests/eval/test_mock_chain.py
index 5391883..d7312b5 100644
--- a/tests/eval/test_mock_chain.py
+++ b/tests/eval/test_mock_chain.py
@@ -41,9 +41,10 @@ def _spec(tmp_path: Path, **overrides) -> dict:
return base
-@pytest.mark.parametrize("mode", ["step", "stage"])
+@pytest.mark.parametrize("mode", ["step", "stage", "goal"])
def test_full_mock_chain_produces_renderable_result_dir(tmp_path, mode):
- episodes = run_from_spec(_spec(tmp_path, harness_config={"execution_mode": mode}))
+ episodes = run_from_spec(_spec(tmp_path, harness_config={"execution_mode": mode},
+ requirements=None if mode == "goal" else []))
assert episodes and episodes[0]["status"] == "scored"
run_root = tmp_path / "run"
episode_dir = run_root / "episodes" / "mock-mock-000"
@@ -64,6 +65,12 @@ def test_full_mock_chain_produces_renderable_result_dir(tmp_path, mode):
assert episode["stop_reason"] == "done" and episode["score"] == 1.0
summary = json.loads((run_root / "summary.json").read_text())
assert summary["scored"] == 1 and summary["average_score"] == 1.0
+ if mode == "goal":
+ memory = json.loads((episode_dir / "task_memory.json").read_text())
+ assert memory["work_items"][0]["status"] == "done"
+ events = [json.loads(line) for line in (episode_dir / "events.jsonl").read_text().splitlines()]
+ assert any(e.get("route") == "stage_verified" for e in events)
+ assert any(e.get("route") == "observation_refresh" for e in events)
# 渲染(demo 占位)可离线打开
page = render_summary.render(run_root)
diff --git a/tests/eval/test_osworld_env.py b/tests/eval/test_osworld_env.py
index 10fe121..98e516c 100644
--- a/tests/eval/test_osworld_env.py
+++ b/tests/eval/test_osworld_env.py
@@ -62,6 +62,30 @@ def test_png_size_parses_ihdr():
assert png_size(_png(1920, 1080)) == (1920, 1080)
+def test_read_only_refresh_keeps_cdp_receipt_and_prior_evidence(tmp_path, env, monkeypatch):
+ snapshot = {"elements": [{"index": 1, "role": "entry", "name": "Receipt Date"}]}
+ monkeypatch.setattr("tandem_eval.osworld.capture_browser_snapshot",
+ lambda controller: (snapshot, {"status": "ok"}))
+ task = _task_file(tmp_path)
+ first = env.reset(task)
+ budget = Budget(steps_remaining=10, wall_seconds_remaining=60)
+ receipt, previous = env.execute(GuiAction("click", {"x": 1, "y": 2}), budget=budget, task=task)
+ steps, commands = list(env.env.steps), list(env.env.controller.commands)
+ before = Path(previous.screenshot.path).read_bytes()
+ refreshed = env.observe(task, budget=budget, previous=previous)
+ assert refreshed.protocol_version == "0.2" and refreshed.last_receipt is receipt
+ assert refreshed.step_idx == previous.step_idx
+ assert env.env.steps == steps and env.env.controller.commands == commands
+ assert refreshed.browser_snapshot.source == "cdp"
+ assert json.loads(Path(refreshed.browser_snapshot.path).read_text()) == snapshot
+ assert refreshed.screenshot.path != previous.screenshot.path != first.screenshot.path
+ assert Path(previous.screenshot.path).read_bytes() == before
+
+
+def test_right_click_codegen_uses_real_mouse_button():
+ assert gui_code(GuiAction("right_click", {"x": 20, "y": 30})) == "pyautogui.rightClick(x=20, y=30)"
+
+
# ------------------------------------------------------------------ adapter
class FakeController:
def __init__(self):
diff --git a/tests/harness/conftest.py b/tests/harness/conftest.py
index f7317ab..e91bc14 100644
--- a/tests/harness/conftest.py
+++ b/tests/harness/conftest.py
@@ -7,6 +7,7 @@
from pathlib import Path
import pytest
+from PIL import Image
from tandem.components import Components
from tandem.gui.candidates import ActionCandidateBuilder
@@ -61,7 +62,7 @@ def make_observation(tmp_path: Path, step_idx: int, *, receipt: Receipt | None =
tree_xml: str | None = A11Y_XML, steps: int = 50,
wall: float = 1800.0, screenshot_sha: str = SHA) -> Observation:
shot = tmp_path / f"step_{step_idx:02d}.png"
- shot.write_bytes(b"png-bytes")
+ Image.new("RGB", (64, 64), "white").save(shot)
a11y_path = None
if tree_xml is not None:
a11y_path = str(tmp_path / f"a11y_{step_idx:02d}.xml")
diff --git a/tests/harness/test_goal_execution.py b/tests/harness/test_goal_execution.py
new file mode 100644
index 0000000..1e94840
--- /dev/null
+++ b/tests/harness/test_goal_execution.py
@@ -0,0 +1,201 @@
+"""Synthetic regressions derived from real failure patterns; not benchmark acceptance."""
+
+from unittest.mock import Mock
+
+import pytest
+
+from tandem.gui.compile import gui_action
+from tandem.gui.goal_candidates import foreground_tree, goal_candidates
+from tandem.gui.targets import A11yTargetBuilder
+from tandem.memory.context import planning_context
+from tandem.models.ranking import Ranking
+from tandem.planning.goal_planner import goal_error, plan_goal
+from tandem.runtime.goal_stage import (
+ ObservationRequired,
+ _action_key,
+ _cycle_actions,
+ _loop_state,
+ _target_effect,
+)
+from tandem.verification.goal_verifier import result_error
+
+from .conftest import A11Y_XML, build_space, build_targets, gui_receipt, make_observation
+from .test_goal_memory import fact, session
+from .test_goal_stage import plan, requirements, result
+
+
+def _space(tree, inputs=()):
+ tree = foreground_tree(tree)
+ targets = A11yTargetBuilder().build(tree, "test", 128, True, True, filter_occluded=True)
+ return targets, goal_candidates(build_space(targets), targets, tree, inputs)
+
+
+def test_receipt_path_detects_a_b_a_b_without_manually_seeding_attempts(tmp_path):
+ s = session()
+ s.begin("Edit", requirements())
+ s.goal.memory.start(plan(), 0)
+ chosen = []
+
+ def rank(**kw):
+ candidates = kw["action_space"].candidates()
+ toggles = [c for c in candidates if c.operation == "click" and "启用" in c.description]
+ if len(chosen) == 4:
+ assert not toggles
+ assert kw["action_space"].controls["return_to_planner"]
+ winner = next(c for c in candidates if c.id == "KEYPRESS_ESCAPE")
+ else:
+ winner = toggles[0]
+ chosen.append(winner)
+ return Ranking(winner, [], [], 0, 0, 1, True)
+
+ s.components.selector.rank = rank
+ for step in range(5):
+ tree = A11Y_XML.replace('checked="false"', f'checked="{str(step % 2 == 1).lower()}"')
+ s.act(make_observation(tmp_path, step, tree_xml=tree,
+ receipt=gui_receipt() if step else None))
+ assert len(s.goal.failed_actions) == 1 # Block current state; the other state blocks on return.
+ assert s.goal.memory.execution_constraints
+ assert planning_context(s.goal.memory)["execution_constraints"]
+ assert s.goal.attempts[0]["state"] == s.goal.attempts[2]["state"]
+
+
+def test_duplicate_popup_nodes_do_not_remove_their_own_close_button():
+ popup = (''
+ ''
+ '')
+ tree = (''
+ ''
+ + popup * 2 + '')
+ targets = A11yTargetBuilder().build(tree, "test", filter_occluded=True)
+ assert [t.name for t in targets].count("Close") == 2
+ assert "PDF" not in [t.name for t in targets]
+ assert "Not covered" in [t.name for t in targets]
+
+
+def test_explicit_popup_menu_excludes_the_covered_media_button():
+ tree = (''
+ '')
+ ts, space = _space(tree)
+ assert "More" not in [t.name for t in ts]
+ assert any("Exit picture-in-picture" in c.description for c in space.target_heads["click"])
+
+
+def test_browser_memory_and_playback_badges_do_not_create_new_loop_states():
+ def state(name):
+ tree = ('')
+ return _loop_state(build_targets(tree), tree)
+
+ assert state("Video - Memory usage - 47.1 MB") == state(
+ "Video – Audio playing - Memory usage - 108 MB")
+ assert state("Different page - Memory usage - 108 MB") != state("Video - Memory usage - 47.1 MB")
+
+
+def test_four_state_cycle_is_detected_but_continuous_scroll_is_not():
+ states = ["closed", "menu", "pip", "pip-menu"]
+ attempts = [{"work": "w", "state": state, "key": ("click", "menu", "")}
+ for state in states * 2]
+ assert _cycle_actions(attempts, "closed") == {("click", "menu", "")}
+ assert not _cycle_actions([{"state": "page", "key": ("scroll", "down", "")}] * 8, "page")
+
+
+def test_overview_excludes_stale_app_and_keeps_unnamed_search_then_typing():
+ tree = (''
+ ''
+ ''
+ ''
+ '')
+ targets, space = _space(tree)
+ assert "Play" not in [t.name for t in targets]
+ assert any("Desktop search" in c.description for c in space.target_heads["click"])
+ assert not space.target_heads["type_text"] # Must focus the observed field first.
+ _, space = _space(tree.replace('focused="false"', 'focused="true"'), [
+ {"label": "Application", "target_hint": "Desktop search", "text": "Shotcut",
+ "input_mode": "replace"}])
+ assert gui_action(space.target_heads["type_text"][0], None).parameters["text"] == "Shotcut"
+
+
+def test_hidden_overview_does_not_take_over_the_foreground():
+ tree = (''
+ ''
+ '')
+ assert 'name="app"' in foreground_tree(tree)
+
+
+def test_slider_has_bounded_distinct_parameters_and_checks_the_intended_value():
+ tree = ('')
+ targets, space = _space(tree)
+ choices = space.target_heads["set_slider"]
+ assert len(choices) == 4 and not space.target_heads["click"]
+ assert len({_action_key(c, targets) for c in choices}) == 4
+ winner = next(c for c in choices if c.metadata["fraction"] == 0.75)
+ assert gui_action(winner, targets[0]).parameters["x"] == 850
+ pending = {"operation": "set_slider", "target_semantic": targets[0].semantic_key, "fraction": 0.75}
+ assert _target_effect(pending, targets) == "not_met"
+ updated, _ = _space(tree.replace('value="50"', 'value="75"'))
+ assert _target_effect(pending, updated) == "met"
+
+
+def test_new_source_fact_does_not_clear_a_failed_gui_action(tmp_path):
+ s = session()
+ s.begin("Edit", requirements())
+ s.goal.memory.start(plan(), 0)
+ s.goal.memory.source_keys.add("quota")
+ s.goal.failed_actions = {("R1:unit:1", "state", "candidate"): 2}
+ s.goal.memory.execution_constraints = [{"work_item_id": "R1:unit:1"}]
+ s.components.stage_verifier = Mock(return_value=result(
+ unit_met=False, stage_outcome="not_met", delivery_state="none", evidence_refs=["S1:O2"],
+ facts=[fact(evidence_refs=["S1:O2"])]))
+ s.goal.end_reason = "returned_to_planner"
+ with pytest.raises(ObservationRequired):
+ s.goal._finish(s, make_observation(tmp_path, 2))
+ assert s.goal.memory.facts["quota"]["value"] == "1400000 USD"
+ assert s.goal.failed_actions and s.goal.memory.execution_constraints
+
+
+def test_planner_application_handoff_supplies_actual_typable_input(tmp_path):
+ s = session()
+ s.begin("Open application", requirements())
+
+ class Client:
+ def json_request(self, messages, validate, **kwargs):
+ value = {**plan(), "target_application": "Shotcut"}
+ assert validate(value)
+ return value
+
+ shot = tmp_path / "screen.png"
+ shot.write_bytes(b"synthetic")
+ value = plan_goal(Client(), "Open application", shot, [], [], "", memory=s.goal.memory)
+ assert value["inputs"][-1]["text"] == "Shotcut"
+
+
+def test_contract_repair_names_the_actual_field_or_semantic_failure():
+ s = session()
+ s.begin("Edit", requirements())
+ assert goal_error({**plan(), "work_item_id": "R9"}, s.goal.memory, True).startswith("work_item_id")
+ assert goal_error({**plan(), "inputs": [{}]}, s.goal.memory, True).startswith("inputs[0]")
+ assert goal_error({**plan(), "work_item_id": {}}, s.goal.memory, True).startswith("work_item_id")
+ missing = plan()
+ missing.pop("command")
+ assert goal_error(missing, s.goal.memory, True).startswith("command")
+ item = s.goal.memory.work_items["R1:unit:1"]
+ item.update(stalled_stages=2, last_attempt={"route": "cli", "command": "same"})
+ assert "repeats" in goal_error({**plan(), "route": "cli", "command": "same"}, s.goal.memory, True)
+ assert result_error(result(facts=[{**fact(), "value": {}}])).startswith("facts[0].value")
+ assert result_error(result(stage_outcome=[])).startswith("stage_outcome")
+ assert result_error(result(materials=[{"source_ref": "x", "summary": "x", "evidence_refs": []}]
+ )).startswith("materials[0].covered_parts")
+
+
+def test_different_fields_and_changed_pages_are_not_the_same_failed_action():
+ first = build_targets()
+ second = build_targets(A11Y_XML.replace("确定", "Next page"))
+ one = build_space(first).target_heads["click"][0]
+ two = build_space(second).target_heads["click"][0]
+ assert _action_key(one, first) != _action_key(two, second)
diff --git a/tests/harness/test_goal_integration.py b/tests/harness/test_goal_integration.py
new file mode 100644
index 0000000..03ab9d6
--- /dev/null
+++ b/tests/harness/test_goal_integration.py
@@ -0,0 +1,114 @@
+"""Synthetic compatibility tests against main's semantic pipeline and JEV recovery."""
+
+import base64
+import io
+import json
+import urllib.error
+from dataclasses import replace
+from unittest.mock import Mock, patch
+
+import pytest
+from PIL import Image
+
+from tandem.config import SessionConfig
+from tandem.memory.state import TaskMemory
+from tandem.models.jev import JevOmniPolicy
+from tandem.models.ranking import Ranking
+from tandem.models.slow import SlowModelClient
+from tandem.protocol.models import BrowserSnapshot
+from tandem.runtime.goal_stage import ObservationRequired
+
+from .conftest import build_space, build_targets, make_observation
+from .test_goal_memory import session
+from .test_goal_stage import plan, requirements
+from .test_selector_compile import ScriptedOpener
+from .test_slow_transport import _Response
+
+
+def test_goal_passes_cdp_enriched_semantics_to_planner_and_selector(tmp_path):
+ s = session()
+ s.begin("Enter the receipt date", requirements())
+ tree = (''
+ ''
+ ''
+ '')
+ observation = make_observation(tmp_path, 0, tree_xml=tree)
+ snapshot = tmp_path / "browser.json"
+ snapshot.write_text(json.dumps({"elements": [{
+ "index": 1, "role": "entry", "name": "Receipt Date", "description": "dd/mm/yyyy",
+ "value": "", "x": 250, "y": 293, "width": 218, "height": 18,
+ "label_source": "dom_label", "operations": ["click", "type_text"],
+ }]}))
+ observation = replace(observation, browser_snapshot=BrowserSnapshot(
+ present=True, source="cdp", path=str(snapshot), node_count=1))
+
+ def planner(*args, **kwargs):
+ target = next(t for t in args[2] if t.name == "Receipt Date")
+ assert target.source == "dom" and target.scope == "web_content"
+ return {**plan(), "inputs": [{"label": "Receipt Date", "target_hint": "Receipt Date",
+ "text": "11/12/2025", "input_mode": "replace"}]}
+
+ s.components.planner = planner
+ with pytest.raises(ObservationRequired, match="planner_returned"):
+ s.act(observation)
+
+ def rank(**kwargs):
+ winner = kwargs["action_space"].target_heads["type_text"][0]
+ target = next(t for t in kwargs["targets"] if t.id == winner.target_id)
+ assert target.name == "Receipt Date" and target.description == "dd/mm/yyyy"
+ return Ranking(winner, [], [], 0, 0, 1, True)
+
+ s.components.selector.rank = rank
+ action = s.act(observation)
+ assert action.action_type == "type" and action.parameters["click"] == [359, 302]
+
+
+def test_joint_goal_choices_retain_main_endpoint_recovery(monkeypatch):
+ monkeypatch.setattr("tandem.models.jev.time.sleep", lambda seconds: None)
+ targets = build_targets()
+ space = build_space(targets)
+ space.target_heads = {"click": space.target_heads["click"][:1]}
+ space.controls = {"done": space.controls["done"]}
+ click, done = space.candidates()
+ opener = ScriptedOpener([
+ urllib.error.URLError("old endpoint unavailable"), {"status": "ok"},
+ {"scores": [0.9, 0.1], "used_image": True}, {"status": "ok"},
+ {"scores": [0.1, 0.9], "used_image": True},
+ ])
+ endpoints = iter(["http://old", "http://new", "http://new"])
+ policy = JevOmniPolicy("http://old", opener=opener, joint_actions=True,
+ endpoint_provider=lambda: next(endpoints))
+ ranking = policy.rank(stage_goal="Goal", intent="Intent", active_window="Chrome",
+ targets=targets, action_space=space, recent_actions=[])
+ assert ranking.operation_ranked[0]["operation"] == click.operation
+ assert ranking.winner is done and policy.endpoint == "http://new"
+ assert opener.requests[-2:] == ["http://new/", "http://new/rank"]
+
+
+def test_goal_image_budget_and_verify_override_do_not_change_step_transport(tmp_path):
+ encoded = io.BytesIO()
+ Image.new("RGB", (100, 100), "white").save(encoded, format="PNG")
+ image_url = "data:image/png;base64," + base64.b64encode(encoded.getvalue()).decode()
+ for mode in ("step", "goal"):
+ config = SessionConfig(execution_mode=mode, slow_max_tokens=32768,
+ slow_verify_max_tokens=8192, slow_reasoning_effort="max",
+ slow_verify_reasoning_effort="low")
+ client = SlowModelClient({"base_url": "https://test.invalid"}, config, tmp_path / "audit.jsonl")
+ messages = [{"role": "user", "content": [{"type": "image_url",
+ "image_url": {"url": image_url}}]}]
+ with patch("urllib.request.urlopen", return_value=_Response()) as opener:
+ client.json_request(messages, bool, role="verify_stage")
+ payload = json.loads(opener.call_args.args[0].data)
+ sent_url = payload["messages"][0]["content"][0]["image_url"]["url"]
+ assert sent_url.startswith("data:image/jpeg" if mode == "goal" else "data:image/png")
+ assert payload["max_tokens"] == 8192 and payload["reasoning_effort"] == "low"
+
+
+def test_goal_memory_and_stage_verifier_are_replaceable():
+ s = session()
+ factory = Mock(return_value=TaskMemory("Edit", requirements()))
+ s.components.memory_factory = factory
+ s.components.stage_verifier = Mock()
+ s.begin("Edit", requirements())
+ factory.assert_called_once_with("Edit", requirements())
+ assert s.goal.memory is factory.return_value
diff --git a/tests/harness/test_goal_memory.py b/tests/harness/test_goal_memory.py
new file mode 100644
index 0000000..de4c155
--- /dev/null
+++ b/tests/harness/test_goal_memory.py
@@ -0,0 +1,265 @@
+"""Synthetic task-memory and local recovery regressions, not benchmark results."""
+
+from unittest.mock import Mock, patch
+
+import pytest
+from PIL import Image
+
+from tandem.config import SessionConfig
+from tandem.gui.candidates import ActionCandidate, ActionCandidateBuilder
+from tandem.gui.goal_candidates import foreground_tree, goal_candidates, scroll_point
+from tandem.gui.targets import A11yTargetBuilder
+from tandem.memory.state import TaskMemory
+from tandem.models.ranking import Ranking
+from tandem.protocol.models import GuiAction
+from tandem.runtime.goal_stage import ObservationRequired
+from tandem.runtime.session import TandemSession
+from tandem_eval.osworld import gui_code
+
+from .conftest import FixedSelector, gui_receipt, make_components, make_observation
+from .test_goal_stage import plan, requirements, result
+
+
+def fact(key="quota", value="1400000 USD", **fields):
+ return {"key": key, "value": value, "status": "confirmed", "scope": "stage",
+ "source_ref": "source.pdf page 1", "evidence_refs": ["A2"], **fields}
+
+
+def source_memory():
+ req = requirements()[0]
+ req.update(facts_needed=[{"key": "quota", "description": "Annual quota"}],
+ unit_fact_keys=[["quota"]])
+ return TaskMemory("Edit", [req])
+
+
+def prepare(memory, update):
+ memory.start({**plan(), "work_item_id": "R1:prepare", "purpose": "prepare"}, 0)
+ memory.apply(result(stage_outcome="not_met", unit_met=False, delivery_state="none",
+ facts=[update]), {"A2": {}})
+
+
+def test_registered_source_is_reusable_even_if_verifier_labels_it_stage():
+ memory = source_memory()
+ prepare(memory, fact())
+ assert memory.facts["quota"]["scope"] == "task"
+ assert memory.work_items["R1:prepare"]["status"] == "done"
+ assert memory.ready_items()[0]["id"] == "R1:unit:1"
+
+
+def test_uncertain_observation_and_unexplained_change_preserve_source():
+ memory = source_memory()
+ prepare(memory, fact())
+ for update in [fact(value="Not currently visible", status="uncertain"), fact(value="Guess")]:
+ memory.start(plan(), 1)
+ memory.apply(result(unit_met=False, delivery_state="none", facts=[update]), {"A2": {}})
+ assert memory.facts["quota"]["value"] == "1400000 USD"
+ assert memory.last_rejections[0]["reason"] == "confirmed_source_preserved"
+ memory.start(plan(), 2)
+ memory.apply(result(unit_met=False, delivery_state="none",
+ facts=[fact(value="1500000 USD", correction_reason="Source correction observed")]),
+ {"A2": {}})
+ assert memory.facts["quota"]["value"] == "1500000 USD"
+
+
+def test_independent_unit_unlocks_before_all_source_reading_finishes():
+ req = requirements()[0]
+ req.update(units=["Save quota page", "Save sales page"], unit_fact_keys=[["quota"], ["sales"]],
+ facts_needed=[{"key": "quota", "description": "Quota"},
+ {"key": "sales", "description": "Sales"}])
+ memory = TaskMemory("Edit", [req])
+ prepare(memory, fact())
+ ready = {w["id"] for w in memory.ready_items()}
+ assert "R1:unit:1" in ready and "R1:unit:2" in ready
+ assert not memory._facts_ready(memory.work_items["R1:unit:2"])
+ assert memory.work_items["R1:prepare"]["status"] != "done"
+
+
+def test_material_only_update_does_not_reset_delivery_stall():
+ memory = TaskMemory("Edit", requirements())
+ memory.start(plan(), 0)
+ memory.apply(result(stage_outcome="not_met", unit_met=False, delivery_state="none",
+ materials=[{"source_ref": "frame.png", "covered_parts": ["another screenshot"],
+ "summary": "No new required facts", "evidence_refs": ["A2"]}]),
+ {"A2": {}})
+ assert memory.stats["source_parts"] == 1
+ assert memory.stats["stages_without_progress"] == 1
+
+
+def test_unit_can_reuse_source_definition_from_another_requirement():
+ req = requirements()[0]
+ req.update(facts_needed=[{"key": "quota", "description": "Quota"}], unit_fact_keys=[["quota"]])
+ other = {**req, "id": "R2", "facts_needed": []}
+ memory = TaskMemory("Edit", [req, other])
+ assert memory.work_items["R2:unit:1"]["facts_needed"] == req["facts_needed"]
+ assert not memory._facts_ready(memory.work_items["R2:unit:1"])
+ prepare(memory, fact())
+ assert "R2:unit:1" in {w["id"] for w in memory.ready_items()}
+
+
+def test_constraint_defined_source_gets_an_executable_preparation_item():
+ delivery = requirements()[0]
+ delivery["unit_fact_keys"] = [["quota"]]
+ constraint = {**delivery, "id": "R2", "kind": "constraint", "units": [], "unit_fact_keys": [],
+ "facts_needed": [{"key": "quota", "description": "Quota from the source"}]}
+ memory = TaskMemory("Edit", [delivery, constraint])
+ assert {w["id"] for w in memory.ready_items()} == {"R1:prepare", "R1:unit:1"}
+ assert not memory._facts_ready(memory.work_items["R1:unit:1"])
+ assert memory.work_items["R1:prepare"]["facts_needed"] == constraint["facts_needed"]
+ prepare(memory, fact())
+ assert [w["id"] for w in memory.ready_items()] == ["R1:unit:1"]
+
+
+def test_source_can_cite_registered_history_but_completion_needs_current_evidence():
+ memory = source_memory()
+ prepare(memory, fact())
+ memory.start(plan(), 1)
+ memory.apply(result(evidence_refs=["A2"]), {"A3": {}})
+ assert memory.work_items["R1:unit:1"]["status"] != "done"
+ memory.start(plan(), 2)
+ memory.apply(result(evidence_refs=["A2", "A3"]), {"A3": {}})
+ assert memory.work_items["R1:unit:1"]["status"] == "done"
+
+
+def test_named_images_and_native_file_targets_survive_bounded_selection():
+ chrome = ''
+ chrome += ''.join(f''
+ for i in range(150))
+ chrome += ''
+ chrome += ''
+ targets = A11yTargetBuilder().build(chrome, "s", 8, context="Open Fractal tutorial", filter_occluded=True)
+ space = ActionCandidateBuilder().build(targets, context="Open Fractal tutorial")
+ assert any("Fractal tutorial" in c.description for c in space.target_heads["click"])
+ files = '''
+
+
+ '''
+ targets = A11yTargetBuilder().build(files, "s", filter_occluded=True)
+ assert {t.name for t in targets} == {"Desktop", "documents"}
+ space = goal_candidates(ActionCandidateBuilder().build(targets), targets, files, [])
+ assert space.target_heads["double_click"]
+
+
+def test_scroll_point_is_inside_the_small_foreground_window():
+ tree = ''
+ x, y = scroll_point(tree, (1920, 1080))
+ assert 94 < x < 746 and 54 < y < 706
+
+
+def test_x11_foreground_identity_excludes_stale_background_active_app():
+ tree = '''
+
+
+
+ '''
+ filtered = foreground_tree(tree)
+ assert 'name="Google Chrome"' not in filtered
+ assert 'name="wpp"' in filtered and 'name="gnome-shell"' in filtered
+
+
+def session():
+ components = make_components(lambda *a, **kw: plan(), lambda *a, **kw: None, FixedSelector())
+ components.memory_factory = TaskMemory
+ components.stage_verifier = lambda *a, **kw: result(unit_met=False, delivery_state="none")
+ return TandemSession(components, config=SessionConfig(execution_mode="goal"))
+
+
+def test_no_effect_forces_grounding_and_masks_the_failed_candidate(tmp_path):
+ s = session()
+ s.begin("Edit", requirements())
+ grounded = ActionCandidate("visual", "click", "Observed alternate", source="visual_grounder",
+ metadata={"visual_point": [500, 500]})
+ s.components.grounder = Mock(return_value=[grounded])
+ obs = make_observation(tmp_path, 0)
+ with pytest.raises(ObservationRequired):
+ s.act(obs)
+ s.act(obs)
+ s.act(make_observation(tmp_path, 1, receipt=gui_receipt()))
+ failed = s.goal.pending["candidate_key"]
+ obs2 = make_observation(tmp_path, 2, receipt=gui_receipt())
+ with pytest.raises(ObservationRequired, match="local_visual_recovery_returned"):
+ s.act(obs2)
+ s.components.grounder.assert_called_once()
+
+ def rank(**kwargs):
+ from tandem.runtime.goal_stage import _action_key
+
+ candidates = kwargs["action_space"].candidates()
+ assert grounded in candidates
+ assert not any(_action_key(c, kwargs["targets"]) == failed for c in candidates)
+ return Ranking(grounded, [], [], 0, 0, 1, True)
+
+ s.components.selector.rank = rank
+ assert s.act(obs2).action_type == "click"
+
+
+def test_unrelated_animation_does_not_reset_semantic_no_effect(tmp_path):
+ s = session()
+ s.begin("Edit", requirements())
+ obs = make_observation(tmp_path, 0)
+ with pytest.raises(ObservationRequired):
+ s.act(obs)
+ s.act(obs)
+ for step, color in [(1, "black"), (2, "white")]:
+ obs = make_observation(tmp_path, step, receipt=gui_receipt())
+ Image.new("RGB", (64, 64), color).save(obs.screenshot.path)
+ assert s.act(obs).kind == "gui"
+ assert s.goal.no_effect == 2
+ assert all(h["observed_change"] and not h["relevant_change"] for h in s.goal.history)
+
+
+def test_unicode_codegen_uses_utf8_clipboard_not_ascii_typing():
+ text = "Organization Change — 组织调整"
+ action = GuiAction("type", {"text": text, "input_mode": "replace"})
+ gui = Mock()
+ with patch("subprocess.run") as run, patch("time.sleep"):
+ exec(gui_code(action), {"pyautogui": gui})
+ assert run.call_args.kwargs["input"] == text.encode("utf-8")
+ assert "UTF8_STRING" in run.call_args.args[0]
+ gui.write.assert_not_called()
+ gui.hotkey.assert_any_call("ctrl", "v")
+
+
+def test_partial_collection_extends_without_overwriting_or_claiming_complete():
+ memory = source_memory()
+ prepare(memory, fact(value="Partial options", entries={"1": "Humanities"}, complete=False))
+ assert not memory._facts_ready(memory.work_items["R1:unit:1"])
+ prepare(memory, fact(value="Full options", entries={"2": "Social Sciences"}, complete=True))
+ assert memory.facts["quota"]["entries"] == {"1": "Humanities", "2": "Social Sciences"}
+ assert memory.work_items["R1:prepare"]["status"] == "done"
+ memory.start(plan(), 2)
+ memory.apply(result(unit_met=False, facts=[fact(value="Changed", entries={"1": "Wrong"})]), {"A2": {}})
+ assert memory.facts["quota"]["entries"]["1"] == "Humanities"
+ assert memory.last_rejections[0]["reason"] == "confirmed_source_preserved"
+
+
+def test_preparation_targets_one_unit_not_all_materials():
+ req = requirements()[0]
+ req.update(units=["Save quota page", "Save sales page"], unit_fact_keys=[["quota"], ["sales"]],
+ facts_needed=[{"key": "quota", "description": "Quota"},
+ {"key": "sales", "description": "Sales"}])
+ memory = TaskMemory("Edit", [req])
+ memory.start({**plan(), "work_item_id": "R1:prepare", "purpose": "prepare"}, 0)
+ assert memory.active_stage["prepares_unit"] == "R1:unit:1"
+ assert memory.active_stage["fact_targets"] == [{"key": "quota", "description": "Quota"}]
+
+
+def test_snapshot_ids_and_ancestry_are_not_a_new_ui_state():
+ from dataclasses import replace
+
+ from tandem.runtime.goal_stage import _state_key
+
+ target = A11yTargetBuilder().build('''''', "one")[0]
+ changed_path = replace(target, snapshot_id="two", semantic_key="changed tab title|Open")
+ assert _state_key([target]) == _state_key([changed_path])
+
+
+def test_action_buttons_are_not_displaced_by_file_metadata_cells():
+ tree = ''
+ tree += ''.join(f''
+ for n in range(45))
+ tree += ''
+ targets = A11yTargetBuilder().build(tree, "s", 128, context="Open Image")
+ space = ActionCandidateBuilder().build(targets, context="Open Image")
+ button = next(t for t in targets if t.name == "Open")
+ assert any(c.target_id == button.id for c in space.target_heads["click"])
diff --git a/tests/harness/test_goal_progress.py b/tests/harness/test_goal_progress.py
new file mode 100644
index 0000000..bc0cc15
--- /dev/null
+++ b/tests/harness/test_goal_progress.py
@@ -0,0 +1,161 @@
+"""Synthetic regressions for partial delivery, field binding and semantic navigation stalls."""
+
+from unittest.mock import Mock
+
+import pytest
+from PIL import Image
+
+from tandem.gui.compile import gui_action
+from tandem.gui.grounding import ground_visual
+from tandem.memory.context import planning_context
+from tandem.memory.state import TaskMemory
+from tandem.models.ranking import Ranking
+from tandem.runtime.goal_stage import ObservationRequired, _document_mismatch, _target_effect
+
+from .conftest import gui_receipt, make_observation
+from .test_goal_execution import _space
+from .test_goal_memory import fact, prepare, session, source_memory
+from .test_goal_stage import plan, requirements, result
+
+
+def test_partial_sources_allow_delivery_without_claiming_source_or_unit_complete():
+ memory = source_memory()
+ prepare(memory, fact(value="Known first section", complete=False))
+ item = planning_context(memory)["ready_work_items"][0]
+ assert item["id"] == "R1:unit:1" and item["can_start_partial"]
+ assert item["next_acquisition"]["missing_facts"][0]["key"] == "quota"
+ memory.start({**plan(), "completion_criteria": "Known first section saved"}, 1)
+ memory.apply(result(unit_met=False, delivery_state="saved", remaining=["Read remaining section"]),
+ {"A2": {}})
+ assert memory.work_items[item["id"]]["delivery_state"] == "saved"
+ assert memory.work_items[item["id"]]["status"] == "pending"
+ assert memory.work_items["R1:prepare"]["status"] != "done"
+ assert not memory.facts["quota"]["complete"]
+ memory.start({**plan(), "purpose": "prepare"}, 2)
+ assert memory.active_stage["fact_targets"][0]["key"] == "quota"
+
+
+def test_explicit_dependencies_still_apply_and_preparation_stays_with_selected_unit():
+ req = requirements()[0]
+ req.update(units=["First section", "Second section"], unit_fact_keys=[["first"], ["second"]],
+ facts_needed=[{"key": key, "description": key} for key in ("first", "second")])
+ memory = TaskMemory("Edit", [req])
+ second = memory.work_items["R1:unit:2"]
+ second["depends_on"] = ["R1:unit:1"]
+ assert second not in memory.ready_items()
+ second["depends_on"] = []
+ memory.start({**plan(), "work_item_id": second["id"], "purpose": "prepare"}, 0)
+ assert memory.active_stage["prepares_unit"] == second["id"]
+ assert memory.active_stage["fact_targets"][0]["key"] == "second"
+
+
+FORM = '''
+ file:///home/user/research-form.html?type=paper
+
+
+
+ '''
+
+
+def test_text_binds_to_observed_label_instead_of_unrelated_focus():
+ targets, space = _space(FORM, [{"label": "Article title", "target_hint": "Title",
+ "text": "Observed source title", "input_mode": "replace"}])
+ choices = space.target_heads["type_text"]
+ assert len(choices) == 1
+ target = next(t for t in targets if t.id == choices[0].target_id)
+ assert target.name == "Title *" and not target.focused
+ action = gui_action(choices[0], target)
+ assert action.parameters["click"] == [300, 115]
+ pending = {"operation": "type_text", "field_label": "Title *", "input_mode": "replace",
+ "expected_text": "Observed source title"}
+ wrong = FORM.replace('size="(200,30)"/>\n