From 475ea53bbf991a1d2a9f3a6f4d8147b2e6dd7cec Mon Sep 17 00:00:00 2001 From: tandem-demo Date: Thu, 29 Oct 2026 14:12:21 +0800 Subject: [PATCH 01/11] feat: add shared semantic perception for GUI candidates --- DEVIATIONS.md | 9 + README.md | 14 +- docs/CLUSTER_RUNTIME.md | 12 + docs/RESULT_DIR_CONTRACT.md | 5 +- src/tandem/components.py | 7 +- src/tandem/gui/candidates.py | 18 +- src/tandem/gui/proposals.py | 53 +++- src/tandem/gui/semantics.py | 237 +++++++++++++++ src/tandem/gui/targets.py | 42 +-- src/tandem/models/ranking.py | 33 ++- src/tandem/planning/planner.py | 12 +- src/tandem/planning/stage_planner.py | 4 +- src/tandem/protocol/__init__.py | 4 +- src/tandem/protocol/models.py | 26 +- .../protocol/samples/observation_no_a11y.json | 3 +- .../samples/observation_osw1_step0.json | 3 +- .../samples/observation_osw2_cdp.json | 29 ++ .../samples/observation_osw2_jev.json | 3 +- .../protocol/schema/observation.schema.json | 22 +- src/tandem/protocol/validate.py | 29 +- src/tandem/runtime/session.py | 21 +- src/tandem_eval/browser.py | 276 ++++++++++++++++++ src/tandem_eval/entry.py | 2 + src/tandem_eval/osworld.py | 24 ++ src/tandem_eval/recorder.py | 11 +- tests/eval/test_browser.py | 77 +++++ tests/eval/test_mock_chain.py | 2 +- tests/eval/test_osworld_env.py | 14 + tests/eval/test_recorder_contract.py | 23 +- tests/harness/conftest.py | 8 +- tests/harness/test_semantics.py | 128 ++++++++ tests/harness/test_session.py | 9 + tests/harness/test_targets.py | 5 +- tests/protocol/test_golden_samples.py | 1 + tests/protocol/test_validate_rejects.py | 16 +- tests/test_package.py | 2 +- 36 files changed, 1120 insertions(+), 64 deletions(-) create mode 100644 src/tandem/gui/semantics.py create mode 100644 src/tandem/protocol/samples/observation_osw2_cdp.json create mode 100644 src/tandem_eval/browser.py create mode 100644 tests/eval/test_browser.py create mode 100644 tests/harness/test_semantics.py diff --git a/DEVIATIONS.md b/DEVIATIONS.md index cff2574..267f6e9 100644 --- a/DEVIATIONS.md +++ b/DEVIATIONS.md @@ -102,3 +102,12 @@ `Ranking`、`build_choice_requests` 和 `ranking_dict` 从原 JEV 模块移到 `models/ranking.py`,供 JEV/Kev 共用;现有函数体、prompt 和排序语义保持不变。 配置字段、启动命令、消息协议、结果目录以及 step/stage 执行行为均不变。 + +## 共享语义感知(2026-10-29) + +30. 【行为】a11y/DOM 先形成慢模型与快模型共用的语义元素表:恢复邻近标签、分离 + `browser_chrome` / `web_content` / `dialog` / `menu` / `native`,当前输入值不再冒充 + 控件名称。CDP 是 Task 008 等已启用远程调试的 Chrome 的可选增强,失败时回退 a11y; + 协议新增 `browser_snapshot` 文件引用并升至 0.2。候选备选只在提案 scope 内生成, + 不再补入全局 click/keypress;无 grounder 时不提供 visual_fallback。JEV 近期历史只保留 + 动作、目标、结果和状态变化,不再携带完整 verification。 diff --git a/README.md b/README.md index c259da7..709c76b 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ ``` 观察 Observation - 截图 · a11y 树 · 预算 · 上一步回执 + 截图 · a11y 树 · 可选 CDP DOM 快照 · 预算 · 上一步回执 │ ▼ ┌─────────────────────────────────────────────┐ @@ -77,16 +77,17 @@ `TandemSession.act(observation)` 是唯一的决策入口,状态机语义(`runtime/session.py`): -1. **读观察**:`Observation` 含截图引用(path + sha256)、a11y 树引用、预算、上一轮回执。 -2. **建目标**:a11y 树解析成 `UITarget` 列表(角色 / 名称 / 坐标 / 边界框,`gui/targets.py`)。 +1. **读观察**:`Observation` 含截图、a11y 树和可选 CDP DOM 快照引用、预算、上一轮回执。 +2. **建目标**:a11y 树解析成 `UITarget`,`gui/semantics.py` 恢复邻近标签、 + 分离浏览器外壳与网页内容,并合并 a11y/DOM 元素。 3. **慢模型规划**:`planning.planner.plan_step` 产出 plan——`current_goal`、`expected_observation`、 `route`(gui/cli)、`action_proposals`、`task_status`、`done`。 4. **终态分支**:planner 声称完成 → `handle_done`:若需求带 `output_path`,先发一条 CLI 存在性探针(`done_probe`)核对交付物文件是否真的存在,探针通过后才进入终态裁决 (`terminal_decision`);未通过则回到规划重来。 5. **CLI 路由**:`route == "cli"` → 直接下发 `CliAction`(bash -lc,cwd=/home/user,1 条 CLI 计 1 步)。 -6. **建候选空间**:`gui/candidates.py` 由目标展开动作(click / double_click / type / scroll / hotkey…), - 再经 `focused_candidates` 聚焦 + `prune_stalled` 剪枝。 +6. **建候选空间**:`gui/candidates.py` 由目标展开动作,再经 `focused_candidates` + 按慢模型提案和元素 scope 聚焦,经 `prune_stalled` 剪枝;网页提案不会补入浏览器外壳候选。 7. **快模型排序**:两阶段——先选 **operation**(点击还是输入),再在胜出 operation 内选 **action**(具体哪个目标)。 结果带 `operation_margin`(两级置信度差)与 `latency_ms`。 8. **编译动作**:`gui/compile.py` 把 winner + 目标编译成**声明式** `GuiAction` @@ -108,7 +109,7 @@ ## 组件装配 -`components.py` 定义依赖注入边界——九个字段,没有插件框架、没有隐藏分发: +`components.py` 定义依赖注入边界——十个字段,没有插件框架、没有隐藏分发: | 字段 | 职责 | 默认实现 | |---|---|---| @@ -117,6 +118,7 @@ | `requirements` | 需求抽取 | `verification.verifier.task_requirements` | | `selector` | 排序后端 | `models.jev.JevOmniPolicy` / `models.kev.KevPolicy` | | `target_builder` | a11y 树 → 目标 | `gui.targets.A11yTargetBuilder` | +| `semantic_enricher` | a11y/DOM → 共享语义元素表 | `gui.semantics.enrich_targets` | | `candidate_builder` | 目标 → 动作候选 | `gui.candidates.ActionCandidateBuilder` | | `focus_candidates` | 提案聚焦 + 剪枝 | `gui.proposals.focused_candidates` | | `compile_gui` | winner → 声明式动作 | `gui.compile.gui_action` | diff --git a/docs/CLUSTER_RUNTIME.md b/docs/CLUSTER_RUNTIME.md index 5cc04cc..826882e 100644 --- a/docs/CLUSTER_RUNTIME.md +++ b/docs/CLUSTER_RUNTIME.md @@ -295,3 +295,15 @@ worker 侧各段日志: - `--benchmark mock` 走 mock 链路(不起 VM、不碰共享资源),可本地跑; - 真实基准走 OSW1 / OSW2 / OSW2.1,按本文 §3–§7。 + +## 11. 浏览器 CDP 语义快照 + +- `tandem_eval.browser` 在 VM 内通过 loopback 依次探测 `1337` / `9222`,只调用 + `GET /json/list` 与 `Runtime.evaluate`,不重启 Chrome、不激活 tab、不执行页面 mutation。 + 多 tab 时选择 `document.visibilityState=visible` 的页面。 +- 任务必须已由环境以远程调试参数启动 Chrome;Task 008 的 setup 已显式启用 `1337`。 + 其他任务未启用、端口被拒绝或快照异常时只记录诊断并回退 a11y,不让整题失败。 +- 快照经协议 `browser_snapshot` 文件引用进入 harness;episode 正本保存在 + `episodes//browser/step_XX.json`。密码、隐藏和文件输入元素不进入快照。 +- 2026-10-29 Task 008 真机 probe 已验证 `1337`、WebSocket 和 DOM 序列化可用, + 无需额外权限。首次有效快照含 4 个元素;9222 被其他环境组件占用时不影响 1337 采集。 diff --git a/docs/RESULT_DIR_CONTRACT.md b/docs/RESULT_DIR_CONTRACT.md index 904270c..d2e5920 100644 --- a/docs/RESULT_DIR_CONTRACT.md +++ b/docs/RESULT_DIR_CONTRACT.md @@ -1,7 +1,7 @@ # 结果目录契约(Result Directory Contract) > eval ↔ demo/分析 的唯一接口。demo/分析工具只读本契约描述的目录;任何一侧改动必须先改本文档。 -> 契约版本:`v0.2`(M3 后)。tandem_eval.recorder 是唯一写入者。 +> 契约版本:`v0.3`。tandem_eval.recorder 是唯一写入者。 ## run 目录(`rjob_logs//`,唯一落盘位置) @@ -27,11 +27,12 @@ OSWorld 2.1 显式配置替代 judge/用户模拟器时,`code_version.json.res | 文件 | 内容 | 写入时机 | |------|------|---------| | `task.json` | 任务定义完整副本 | episode 开始 | -| `traj.jsonl` | 每执行步一行:`{step_idx, wall_time, protocol_version, observation{screenshot(sha256/path/w/h), accessibility_tree(present/source/digest/node_count), budget(steps_remaining, wall_seconds_remaining), last_receipt}, action{kind, …}}` | 每步 env.execute 后 | +| `traj.jsonl` | 每执行步一行:`{step_idx, wall_time, protocol_version, observation{screenshot(sha256/path/w/h), accessibility_tree(present/source/digest/node_count), browser_snapshot(present/source/digest/node_count/path 或 null), budget(...), last_receipt}, action{kind, …}}` | 每步 env.execute 后 | | `llm_calls.jsonl` | 全量 LLM 请求/响应,每行 `{role, time, request{payload, messages}, response, attempt}`;Bearer/api_key 脱敏 | 慢模型客户端成功响应时(llm_sink → LlmTap → 本文件) | | `events.jsonl` | harness 决策事件(route 字段区分):`planner`(发给 LLM 的计划全文 + 有效响应)/ `gui`(候选列表 + ranking{operation_request, action_request, operation_ranked, action_ranked, winner, margins, latency_ms, used_image} + 编译后动作)/ `cli`(路由决策 + 命令)/ `gui_verified` / `cli_verified`(验证裁决 + 回执)/ `terminal`(终态双确认)/ `action`(runner 写的执行记录,与 session 事件穿插) | session.act() 后增量排水 | | `a11y_digests.jsonl` | 每步 a11y sha256 + 节点数(`{step_idx, sha256, node_count}`) | 每步(全量 xml 默认关闭,record.a11y_full=true 时另写 `a11y/step_XX.xml`) | | `screenshots/step_XX.png` | 逐步截图(**契约正本**) | 每步 env.execute 后 | +| `browser/step_XX.json` | CDP DOM 语义快照(启用且成功时);traj 的 browser_snapshot.path 指向此正本 | 每步 env.execute 后 | | `deliverables/` | finish 时从 VM 导出的交付物文件(含 export_paths) | episode 终态 | | `artifacts.json` | deliverables 导出记录(`[{vm_path, exported, local_path, bytes}]`) | episode 终态 | | `evaluation.json` | 官方评分(`{source: "official_benchmark_evaluator", raw, score}`) | episode 终态(评分成功时) | diff --git a/src/tandem/components.py b/src/tandem/components.py index 8b8bcda..f66765b 100644 --- a/src/tandem/components.py +++ b/src/tandem/components.py @@ -1,6 +1,6 @@ """组件契约(依赖注入边界,无插件框架、无隐藏分发)。 -九个字段名与旧仓库 components.py 保持一致;唯一差异:grounder(visual fallback) -按重构计划 §3.1 延后,默认 None,session 侧判空跳过。compile_gui 现在产出协议动作。""" +在旧九组件上增加 semantic_enricher,统一 slow planner 与 fast selector 看到的元素语义。 +grounder(visual fallback)默认 None,session 侧判空跳过。compile_gui 产出协议动作。""" from __future__ import annotations @@ -12,6 +12,7 @@ from .gui.candidates import ActionCandidateBuilder from .gui.compile import gui_action from .gui.proposals import focused_candidates +from .gui.semantics import enrich_targets from .gui.targets import A11yTargetBuilder from .models.jev import JevOmniPolicy from .planning.planner import plan_step @@ -26,6 +27,7 @@ class Components: requirements: Callable selector: object target_builder: object + semantic_enricher: Callable candidate_builder: object focus_candidates: Callable grounder: Callable | None @@ -52,6 +54,7 @@ def build_default(*, client, endpoint: str, config: SessionConfig, requirements=partial(task_requirements, client), selector=selector, target_builder=A11yTargetBuilder(), + semantic_enricher=enrich_targets, candidate_builder=ActionCandidateBuilder(), focus_candidates=focused_candidates, grounder=None, diff --git a/src/tandem/gui/candidates.py b/src/tandem/gui/candidates.py index e110742..02cb346 100644 --- a/src/tandem/gui/candidates.py +++ b/src/tandem/gui/candidates.py @@ -159,7 +159,7 @@ def candidates(operation: str, values: list[UITarget], description: str) -> list candidates( "type_text", editable_targets, - 'The {role} "{name}" is the goal-required editable field whose contents should be replaced next.', + 'Replace text in the {role} "{name}".', ) if action_inputs is None else [] @@ -186,8 +186,8 @@ def candidates(operation: str, values: list[UITarget], description: str) -> list operation="type_text", target_id=target.id, description=( - f"The single best next action is to replace the contents of the " - f'{target.role.replace("_", " ")} "{target.name}" with {value!r}.' + f'Replace text in the {target.role.replace("_", " ")} ' + f'"{target.name}" with {value!r}.' ), metadata={ "semantic_key": target.semantic_key, @@ -224,15 +224,15 @@ def candidates(operation: str, values: list[UITarget], description: str) -> list target_heads = { "click": candidates( - "click", - click_targets, - 'The {role} "{name}" is the observed control that should be activated next.', + "click", + click_targets, + 'Click the visible {role} "{name}".', ), "type_text": type_candidates, "double_click": candidates( - "double_click", - double_targets, - 'The {role} "{name}" is the observed file or list item that should be opened next.', + "double_click", + double_targets, + 'Double-click the visible {role} "{name}".', ), "keypress": [ ActionCandidate(id=choice_id, operation="keypress", description=description, source="global") diff --git a/src/tandem/gui/proposals.py b/src/tandem/gui/proposals.py index 784c49f..513567a 100644 --- a/src/tandem/gui/proposals.py +++ b/src/tandem/gui/proposals.py @@ -19,6 +19,33 @@ def focused_candidates( focused_inputs = [item for item in focused_inputs if item["role"] != "table"] or focused_inputs if verified_focus: focused_inputs = [verified_focus] + else: + enriched_focus = [] + for item in focused_inputs: + target = next( + ( + candidate + for candidate in targets + if candidate.role == item["role"] + and [candidate.x, candidate.y, candidate.width, candidate.height] == item["bounds"] + ), + None, + ) + enriched_focus.append( + { + **item, + **( + { + "name": target.name, + "label_source": target.label_source, + "scope": target.scope, + } + if target + else {} + ), + } + ) + focused_inputs = enriched_focus def bounds(element): origin = element.attrib.get(f"{{{cp}}}screencoord", "") @@ -96,6 +123,12 @@ def walk(element, region=None): ) elif not proposal.get("target_id") and len(focused_inputs) == 1: focus = focused_inputs[0] + if ( + name + and focus["name"].casefold() != name.casefold() + and focus.get("label_source") not in {None, "unresolved", "synthetic"} + ): + continue if focus["role"] == "terminal" and mode != "insert": continue if mode == "replace" and focus["role"] not in EDITABLE_ROLES | {"combo_box", "paragraph"}: @@ -262,7 +295,12 @@ def walk(element, region=None): elif operation == "wait": chosen.append(space.controls["wait"]) proposed_operations = {candidate.operation for candidate in chosen} - for operation in ("click", "double_click", "keypress"): + proposed_scopes = { + by_id[candidate.target_id].scope + for candidate in chosen + if candidate.target_id in by_id + } + for operation in ("click", "double_click"): if operation not in proposed_operations: continue offered_targets = { @@ -270,11 +308,18 @@ def walk(element, region=None): for candidate in chosen if candidate.operation == operation and candidate.target_id } - for candidate in space.target_heads.get(operation, [])[:8]: - if candidate.target_id and candidate.target_id in offered_targets: + added = 0 + for candidate in space.target_heads.get(operation, []): + if added >= 4: + break + if not candidate.target_id or candidate.target_id in offered_targets: + continue + target = by_id.get(candidate.target_id) + if target is None or proposed_scopes and target.scope not in proposed_scopes: continue chosen.append(candidate) - distinct = list({candidate.id: candidate for candidate in chosen}.values())[:18] + added += 1 + distinct = list({candidate.id: candidate for candidate in chosen}.values())[:12] space.target_heads = {"focused": distinct} space.controls = { key: value for key, value in space.controls.items() if key in {"visual_fallback", "return_to_planner"} diff --git a/src/tandem/gui/semantics.py b/src/tandem/gui/semantics.py new file mode 100644 index 0000000..379ebd9 --- /dev/null +++ b/src/tandem/gui/semantics.py @@ -0,0 +1,237 @@ +"""Shared semantic element table for the slow planner and fast selector.""" + +from __future__ import annotations + +import re +import xml.etree.ElementTree as ET +from dataclasses import replace + +from .targets import UITarget + +_LABEL_ROLES = {"label", "static", "text"} +_MEMORY_USAGE = re.compile(r"\s+-\s+Memory usage\s+-\s+[\d.]+\s+[KMG]B\b", re.I) + + +def _norm(value: str) -> str: + return " ".join(_MEMORY_USAGE.sub("", value or "").lower().split()) + + +def _key(target: UITarget) -> str: + return "|".join((target.role, _norm(target.name), *(_norm(p) for p in target.parent_context))) + + +def _bounds(element: ET.Element) -> tuple[int, int, int, int] | None: + attrs = {key.rsplit("}", 1)[-1]: value for key, value in element.attrib.items()} + values = [int(v) for v in re.findall(r"-?\d+", attrs.get("screencoord", "") + attrs.get("size", ""))] + return tuple(values) if len(values) == 4 else None + + +def _a11y_context(root: ET.Element | None) -> tuple[list[dict], bool]: + if root is None: + return [], True + labels = [] + chrome_active = False + for app in root: + app_active = app.attrib.get("active") == "true" or any( + child.attrib.get("active") == "true" for child in app + ) + chrome_active = chrome_active or ( + app_active and "chrome" in app.attrib.get("name", "").lower() + ) + for element in root.iter(): + role = element.tag.rsplit("}", 1)[-1].replace("-", "_") + if role not in _LABEL_ROLES: + continue + box = _bounds(element) + text = " ".join((element.attrib.get("name") or element.text or "").split()) + if box and text and len(text) <= 120: + x, y, width, height = box + labels.append({"text": text, "box": box, "cx": x + width / 2, "cy": y + height / 2}) + return labels, chrome_active + + +def _nearest_label(target: UITarget, labels: list[dict]) -> str: + if None in (target.x, target.y, target.width, target.height): + return "" + cx, cy = target.x + target.width / 2, target.y + target.height / 2 + best, best_score = "", float("inf") + for item in labels: + x, y, width, height = item["box"] + left = x + width <= target.x + 4 and abs(item["cy"] - cy) <= max(14, target.height) + above = y + height <= target.y + 4 and x + width >= target.x and x <= target.x + target.width + if not left and not above: + continue + score = ( + target.x - x - width + abs(item["cy"] - cy) + if left + else 1000 + target.y - y - height + abs(item["cx"] - cx) / 4 + ) + if score < best_score: + best, best_score = item["text"], score + return best + + +def _content_y(targets: list[UITarget], snapshot: dict | None) -> float: + if snapshot and snapshot.get("viewport", {}).get("screen_y") is not None: + return float(snapshot["viewport"]["screen_y"]) + address = next((item for item in targets if item.name == "Address and search bar"), None) + if address and address.y is not None and address.height is not None: + return address.y + address.height + 8 + return 110.0 + + +def _scope(target: UITarget, content_y: float) -> str: + if target.role in {"menu", "menu_item"}: + return "menu" + parents = " ".join(target.parent_context).lower() + if any(word in parents for word in ("dialog", "file chooser", "restore pages")): + return "dialog" + application = target.application.lower() + if "chrome" not in application and "chromium" not in application: + return "native" + if target.y is not None and target.height is not None and target.y + target.height / 2 >= content_y: + return "web_content" + return "browser_chrome" + + +def _overlap(a: UITarget, b: UITarget) -> float: + wide = min(a.x + a.width, b.x + b.width) - max(a.x, b.x) + tall = min(a.y + a.height, b.y + b.height) - max(a.y, b.y) + smaller = min(a.width * a.height, b.width * b.height) + return wide * tall / smaller if wide > 0 and tall > 0 and smaller > 0 else 0.0 + + +def _roles_compatible(a: str, b: str) -> bool: + groups = ( + {"button", "push_button", "toggle_button"}, + {"text", "textbox", "entry", "searchbox", "password_text"}, + {"tab", "page_tab"}, + {"checkbox", "check_box"}, + {"table_cell", "grid_cell"}, + ) + return a == b or any(a in group and b in group for group in groups) + + +def _dom_target(item: dict, snapshot_id: str) -> UITarget | None: + try: + x, y = round(float(item["x"])), round(float(item["y"])) + width, height = round(float(item["width"])), round(float(item["height"])) + name = str(item["name"]).strip() + except (KeyError, TypeError, ValueError): + return None + if not name or width < 2 or height < 2: + return None + target = UITarget( + id=f"dom:{snapshot_id}:{item.get('index', 0)}", + snapshot_id=snapshot_id, + role=str(item.get("role") or "unknown"), + name=name, + value=str(item.get("value") or ""), + enabled=True, + visible=True, + showing=True, + selected=item.get("selected"), + checked=item.get("checked"), + focused=None, + x=x, + y=y, + width=width, + height=height, + semantic_key="", + parent_context=(), + source="dom", + application="Google Chrome", + description=str(item.get("description") or ""), + nearby_text=str(item.get("nearby_text") or ""), + scope="web_content", + label_source=str(item.get("label_source") or "dom"), + operations=tuple(item.get("operations") or ("click",)), + ) + return replace(target, semantic_key=_key(target)) + + +def _merge(base: UITarget, dom: UITarget) -> UITarget: + use_dom_name = ( + dom.label_source != "dom" + or base.label_source == "unresolved" + or len(dom.name) >= len(base.name) + ) + merged = replace( + base, + name=dom.name if use_dom_name else base.name, + value=dom.value or base.value, + checked=dom.checked if dom.checked is not None else base.checked, + selected=dom.selected if dom.selected is not None else base.selected, + source="a11y+dom", + description=dom.description or base.description, + nearby_text=dom.nearby_text or base.nearby_text, + scope="web_content", + label_source=dom.label_source, + operations=dom.operations, + ) + return replace(merged, semantic_key=_key(merged)) + + +def enrich_targets( + targets: list[UITarget], + accessibility_tree: str | None, + screen_size: tuple[int, int], + browser_snapshot: dict | None = None, +) -> list[UITarget]: + """Recover labels, classify scope, and merge DOM semantics without adding screen OCR.""" + try: + root = ET.fromstring(accessibility_tree) if accessibility_tree else None + except ET.ParseError: + root = None + labels, chrome_active = _a11y_context(root) + content_y = _content_y(targets, browser_snapshot) + enriched = [] + for target in targets: + name, source = target.name, target.label_source + if target.role == "page_tab": + name = _MEMORY_USAGE.sub("", name) + nearby = target.nearby_text or _nearest_label(target, labels) + if not name and nearby: + name, source = nearby, "a11y_nearby" + if not name and target.focused: + name, source = f"Focused {target.role.replace('_', ' ')}", "synthetic" + if not name: + continue + item = replace( + target, + name=name, + nearby_text=nearby, + scope=_scope(target, content_y), + label_source=source, + ) + enriched.append(replace(item, semantic_key=_key(item))) + if browser_snapshot and chrome_active: + snapshot_id = targets[0].snapshot_id if targets else "dom" + for raw in browser_snapshot.get("elements", []): + dom = _dom_target(raw, snapshot_id) + if dom is None: + continue + compatible = [ + (index, _overlap(base, dom)) + for index, base in enumerate(enriched) + if _roles_compatible(base.role, dom.role) + ] + best = max(compatible, key=lambda pair: pair[1], default=(None, 0.0)) + if best[0] is not None and best[1] >= 0.5: + enriched[best[0]] = _merge(enriched[best[0]], dom) + else: + enriched.append(dom) + unique = {} + for target in enriched: + center_x = target.x + target.width / 2 + center_y = target.y + target.height / 2 + if not (0 <= center_x < screen_size[0] and 0 <= center_y < screen_size[1]): + continue + key = ( + target.role, _norm(target.name), round(target.x), round(target.y), + round(target.width), round(target.height), + ) + current = unique.get(key) + if current is None or current.source == "a11y" and target.source != "a11y": + unique[key] = target + return list(unique.values()) diff --git a/src/tandem/gui/targets.py b/src/tandem/gui/targets.py index c65fcd9..0f7bd8a 100644 --- a/src/tandem/gui/targets.py +++ b/src/tandem/gui/targets.py @@ -27,6 +27,12 @@ class UITarget: semantic_key: str parent_context: tuple[str, ...] source: str = "a11y" + application: str = "" + description: str = "" + nearby_text: str = "" + scope: str = "native" + label_source: str = "a11y" + operations: tuple[str, ...] = () def compact(self) -> dict: return {key: value for key, value in asdict(self).items() if value not in (None, "", (), False)} @@ -106,13 +112,17 @@ def build( root = ET.fromstring(raw_tree) targets: list[UITarget] = [] - def visit(element: ET.Element, active: bool, parents: tuple[str, ...]) -> None: + def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name: str) -> None: active = active or _state(element, "active") is True role = _local(element.tag) - name = str(element.attrib.get("name") or "").strip() + raw_name = str(element.attrib.get("name") or "").strip() + name = raw_name + if role == "application": + app_name = name + label_source = "a11y" if name else "unresolved" value = ( str(element.text or "").strip() - if role in {"text", "textbox", "entry", "searchbox", "password_text", "spin_button"} + if role in {"text", "textbox", "entry", "searchbox", "spin_button"} else "" ) position = _pair(_attribute(element, "screencoord")) @@ -121,23 +131,14 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...]) -> None: labels = { str(child.attrib.get("name") or "").strip() for child in element.iter() - if _local(child.tag) == "label" and child.attrib.get("name") + if _local(child.tag) in {"label", "static"} and child.attrib.get("name") } if len(labels) == 1: name = labels.pop() + label_source = "a11y_descendant" enabled = _state(element, "enabled") visible = _state(element, "visible") showing = _state(element, "showing") - if ( - not name - and active - and role in {"textbox", "text", "entry", "searchbox", "password_text", "spin_button"} - and _state(element, "editable") is True - and _state(element, "focused") is True - and position - and size - ): - name = f"Focused {role.replace('_', ' ')}" + (f" (current: {value})" if value else "") context = parents + ((name,) if name else ()) interactive_text = role != "text" or ( _state(element, "editable") is True or _state(element, "focusable") is True @@ -158,7 +159,7 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...]) -> None: if ( (active or system_target) and (role in SUPPORTED_ROLES or safe_label or (include_sliders and role == "slider")) - and name + and (name or role in SUPPORTED_ROLES) and interactive_text and enabled is not False and visible is not False @@ -190,10 +191,17 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...]) -> None: height=size[1], semantic_key="|".join((role, normalized_name, *[p.lower() for p in parent_context])), parent_context=parent_context, + application=app_name, + description=( + _attribute(element, "description") + or _attribute(element, "placeholder") + or _attribute(element, "placeholder-text") + ), + label_source=label_source, ) ) for child in element: - visit(child, active, context[-4:]) + visit(child, active, context[-4:], app_name) - visit(root, False, ()) + visit(root, False, (), "") return targets diff --git a/src/tandem/models/ranking.py b/src/tandem/models/ranking.py index 6c59716..846c6ae 100644 --- a/src/tandem/models/ranking.py +++ b/src/tandem/models/ranking.py @@ -35,17 +35,33 @@ def build_choice_requests( ) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: candidates = action_space.candidates() offered_target_ids = {candidate.target_id for candidate in candidates} + control_fields = { + "id", "role", "name", "description", "value", "focused", "checked", "selected", "application", + "scope", "nearby_text", "parent_context", "x", "y", "width", "height", + } controls = { - target.id: {key: value for key, value in target.compact().items() if key not in {"id", "snapshot_id"}} + target.id: { + key: value + for key, value in target.compact().items() + if key in control_fields and not (key == "value" and target.role == "password_text") + } for target in targets if target.id in offered_target_ids } + recent = [ + { + key: item[key] + for key in ("operation", "target", "goal", "outcome", "state_changed") + if key in item + } + for item in recent_actions[-10:] + ] state = json.dumps( { "local_goal": intent, "active_window": active_window, "current_ui_controls": list(controls.values()), - "recent_semantic_actions": recent_actions[-10:], + "recent_semantic_actions": recent, "stage_goal": stage_goal, }, ensure_ascii=False, @@ -70,8 +86,19 @@ def build_choice_requests( target = controls.get(candidate.target_id) if target: context = " > ".join(target.get("parent_context", [])[-2:]) + details = [] + if target.get("description"): + details.append(f"Details: {target['description'][:160]}") + nearby = target.get("nearby_text", "") + if nearby and nearby.casefold() != target.get("name", "").casefold(): + details.append(f"Nearby: {nearby[:240]}") + if target.get("value"): + details.append(f"Current value: {str(target['value'])[:160]}") + if target.get("scope"): + details.append(f"Scope: {target['scope']}") description += ( - f" Parent context: {context};" + (" " + "; ".join(details) + ";" if details else "") + + f" Parent context: {context};" f" screen box ({target.get('x')}, {target.get('y')}," f" {target.get('width')}, {target.get('height')})." ) diff --git a/src/tandem/planning/planner.py b/src/tandem/planning/planner.py index 94a1d7e..6e7b1ab 100644 --- a/src/tandem/planning/planner.py +++ b/src/tandem/planning/planner.py @@ -39,7 +39,7 @@ def plan_step( key=lambda item: len(set(re.findall(r"[a-z]+|\d+", item["path"].lower())) & context_words), reverse=True, )[:32] - planner_targets = sorted( + ordered_targets = sorted( enumerate(targets), key=lambda pair: ( len(set(re.findall(r"[a-z]+|\d+", pair[1].name.lower())) & context_words), @@ -47,7 +47,11 @@ def plan_step( -pair[0], ), reverse=True, - )[:32] + ) + planner_targets = [pair for pair in ordered_targets if pair[1].scope != "browser_chrome"][:28] + planner_targets += [pair for pair in ordered_targets if pair[1].scope == "browser_chrome"][:4] + selected = {id(pair) for pair in planner_targets} + planner_targets += [pair for pair in ordered_targets if id(pair) not in selected][: 32 - len(planner_targets)] prompt = { "task": instruction, "requirements": requirements or [], @@ -59,6 +63,10 @@ def plan_step( "name": target.name, "bounds": [target.x, target.y, target.width, target.height], "focused": bool(target.focused), + "description": target.description, + "value": target.value, + "scope": target.scope, + "nearby_text": target.nearby_text, } for _, target in planner_targets ], diff --git a/src/tandem/planning/stage_planner.py b/src/tandem/planning/stage_planner.py index 57095ca..3deaaa9 100644 --- a/src/tandem/planning/stage_planner.py +++ b/src/tandem/planning/stage_planner.py @@ -98,7 +98,9 @@ def plan_stage(client, instruction, screenshot, targets, recent, accessibility_t "visible_controls": [ {"name": target.name, "role": target.role, "value": target.value, "parents": target.parent_context, "focused": target.focused, - "checked": target.checked, "selected": target.selected} for target in targets + "checked": target.checked, "selected": target.selected, + "description": target.description, "scope": target.scope, + "nearby_text": target.nearby_text} for target in targets ], "focused_inputs": [verified_focus] if verified_focus else inputs, "menu_paths": menus, "recent_actions_and_evidence": recent[-6:], diff --git a/src/tandem/protocol/__init__.py b/src/tandem/protocol/__init__.py index 9b5d6e8..382f020 100644 --- a/src/tandem/protocol/__init__.py +++ b/src/tandem/protocol/__init__.py @@ -1,4 +1,4 @@ -"""消息层协议 v0.1:Observation / Action / Receipt。 +"""消息层协议 v0.2:Observation / Action / Receipt。 真源:schema/*.json;标准样例:samples/*.json;运行时校验:validate.py。 """ @@ -8,6 +8,7 @@ PROTOCOL_VERSION, AccessibilityTree, Action, + BrowserSnapshot, Budget, CliAction, FailAction, @@ -30,6 +31,7 @@ "PROTOCOL_VERSION", "GUI_ACTION_TYPES", "AccessibilityTree", + "BrowserSnapshot", "Action", "Budget", "CliAction", diff --git a/src/tandem/protocol/models.py b/src/tandem/protocol/models.py index d772f30..e9df713 100644 --- a/src/tandem/protocol/models.py +++ b/src/tandem/protocol/models.py @@ -1,4 +1,4 @@ -"""协议 v0.1 消息模型。 +"""协议 v0.2 消息模型。 schema/ 下同名 JSON Schema 是唯一真源;本文件是它的 Python 包装层。 约束(MCP-ready,自第一天生效): @@ -12,7 +12,7 @@ from dataclasses import dataclass, field from typing import Any -PROTOCOL_VERSION = "0.1" +PROTOCOL_VERSION = "0.2" # GUI 动作类型枚举。v0.1 为暂定集,M1 移植 compile 语义时如有出入以移植结果为准并 bump 版本。 GUI_ACTION_TYPES = ( @@ -70,6 +70,26 @@ def to_dict(self) -> dict[str, Any]: } +@dataclass(frozen=True, slots=True) +class BrowserSnapshot: + """可选的浏览器 DOM 快照引用;CDP 不可用时为 None。""" + + present: bool + source: str # "cdp" | "none" + digest: str | None = None + node_count: int | None = None + path: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "present": self.present, + "source": self.source, + "digest": self.digest, + "node_count": self.node_count, + "path": self.path, + } + + @dataclass(frozen=True, slots=True) class Budget: steps_remaining: int @@ -115,6 +135,7 @@ class Observation: screenshot: Screenshot accessibility_tree: AccessibilityTree last_receipt: Receipt | None + browser_snapshot: BrowserSnapshot | None = None protocol_version: str = PROTOCOL_VERSION def to_dict(self) -> dict[str, Any]: @@ -126,6 +147,7 @@ def to_dict(self) -> dict[str, Any]: "screenshot": self.screenshot.to_dict(), "accessibility_tree": self.accessibility_tree.to_dict(), "last_receipt": self.last_receipt.to_dict() if self.last_receipt is not None else None, + "browser_snapshot": self.browser_snapshot.to_dict() if self.browser_snapshot else None, "protocol_version": self.protocol_version, } diff --git a/src/tandem/protocol/samples/observation_no_a11y.json b/src/tandem/protocol/samples/observation_no_a11y.json index 305360e..6db4e5d 100644 --- a/src/tandem/protocol/samples/observation_no_a11y.json +++ b/src/tandem/protocol/samples/observation_no_a11y.json @@ -25,5 +25,6 @@ "acknowledged": true, "full_output_ref": null }, - "protocol_version": "0.1" + "browser_snapshot": null, + "protocol_version": "0.2" } diff --git a/src/tandem/protocol/samples/observation_osw1_step0.json b/src/tandem/protocol/samples/observation_osw1_step0.json index b8d1a69..7c4c6cd 100644 --- a/src/tandem/protocol/samples/observation_osw1_step0.json +++ b/src/tandem/protocol/samples/observation_osw1_step0.json @@ -18,5 +18,6 @@ "path": "a11y/step_00.json" }, "last_receipt": null, - "protocol_version": "0.1" + "browser_snapshot": null, + "protocol_version": "0.2" } diff --git a/src/tandem/protocol/samples/observation_osw2_cdp.json b/src/tandem/protocol/samples/observation_osw2_cdp.json new file mode 100644 index 0000000..2b868e8 --- /dev/null +++ b/src/tandem/protocol/samples/observation_osw2_cdp.json @@ -0,0 +1,29 @@ +{ + "task_id": "osw21-008-expense-report", + "instruction": "Submit the reimbursement claim.", + "step_idx": 3, + "budget": { "steps_remaining": 27, "wall_seconds_remaining": 1500.0 }, + "screenshot": { + "path": "obs_stage/008/step_03.png", + "sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "width": 1920, + "height": 1080, + "captured": true + }, + "accessibility_tree": { + "present": true, + "source": "vm", + "digest": "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", + "node_count": 420, + "path": "obs_stage/008/step_03.xml" + }, + "last_receipt": null, + "browser_snapshot": { + "present": true, + "source": "cdp", + "digest": "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", + "node_count": 18, + "path": "obs_stage/008/step_03.dom.json" + }, + "protocol_version": "0.2" +} diff --git a/src/tandem/protocol/samples/observation_osw2_jev.json b/src/tandem/protocol/samples/observation_osw2_jev.json index 266b9b9..88a3868 100644 --- a/src/tandem/protocol/samples/observation_osw2_jev.json +++ b/src/tandem/protocol/samples/observation_osw2_jev.json @@ -25,5 +25,6 @@ "acknowledged": true, "full_output_ref": "artifacts/step_11_cli_full.txt" }, - "protocol_version": "0.1" + "browser_snapshot": null, + "protocol_version": "0.2" } diff --git a/src/tandem/protocol/schema/observation.schema.json b/src/tandem/protocol/schema/observation.schema.json index c82674b..8bb51e5 100644 --- a/src/tandem/protocol/schema/observation.schema.json +++ b/src/tandem/protocol/schema/observation.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "tandem:protocol/observation.schema.json", - "title": "Tandem Observation v0.1", + "title": "Tandem Observation v0.2", "description": "环境 → harness 的观察消息。二进制一律文件引用,禁止内嵌(MCP-ready 约束)。", "type": "object", "additionalProperties": false, @@ -60,9 +60,27 @@ }, "description": "一致性:present=true ⇒ source∈(vm,jev) 且 digest/node_count 必填;present=false ⇒ 其余字段为 null。" }, + "browser_snapshot": { + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "additionalProperties": false, + "required": ["present", "source"], + "properties": { + "present": { "type": "boolean" }, + "source": { "enum": ["cdp", "none"] }, + "digest": { "type": ["string", "null"], "minLength": 1 }, + "node_count": { "type": ["integer", "null"], "minimum": 0 }, + "path": { "type": ["string", "null"], "minLength": 1 } + }, + "description": "CDP DOM 快照文件引用;不可用时为 null。" + } + ] + }, "last_receipt": { "oneOf": [{ "type": "null" }, { "$ref": "tandem:protocol/receipt.schema.json" }] }, - "protocol_version": { "const": "0.1" } + "protocol_version": { "const": "0.2" } } } diff --git a/src/tandem/protocol/validate.py b/src/tandem/protocol/validate.py index eb21ecf..0e97313 100644 --- a/src/tandem/protocol/validate.py +++ b/src/tandem/protocol/validate.py @@ -1,4 +1,4 @@ -"""协议 v0.1 手写校验器(零依赖)。 +"""协议 v0.2 手写校验器(零依赖)。 与 schema/*.json 的关系:schema 文件是对外(人类阅读 / 未来 MCP tool 定义)的真源; 本文件是运行时的强制执行者。二者与 samples/ 由 tests/protocol 的 golden 测试共同锚定。 @@ -20,6 +20,7 @@ WANT_OBSERVATION_VALUES, AccessibilityTree, Action, + BrowserSnapshot, Budget, CliAction, FailAction, @@ -33,6 +34,7 @@ RECEIPT_KINDS = ("cli", "gui", "wait") A11Y_SOURCES = ("vm", "jev", "none") +BROWSER_SNAPSHOT_SOURCES = ("cdp", "none") _SHA256_RE = re.compile(r"^[0-9a-f]{64}$") @@ -190,6 +192,26 @@ def _a11y_from_dict(data: Any, path: str) -> AccessibilityTree: ) +def _browser_snapshot_from_dict(data: Any, path: str) -> BrowserSnapshot | None: + if data is None: + return None + obj = _as_obj(data, path) + _take(obj, ["present", "source"], ["digest", "node_count", "path"], path) + present = _as_bool(obj["present"], f"{path}.present") + source = _as_enum(obj["source"], BROWSER_SNAPSHOT_SOURCES, f"{path}.source") + digest = _as_nullable(obj.get("digest"), lambda v, p: _as_str(v, p), f"{path}.digest") + node_count = _as_nullable( + obj.get("node_count"), lambda v, p: _as_int(v, p, minimum=0), f"{path}.node_count" + ) + snapshot_path = _as_nullable(obj.get("path"), lambda v, p: _as_str(v, p), f"{path}.path") + if present: + if source != "cdp" or digest is None or node_count is None or snapshot_path is None: + _fail(path, "present=true 时 source=cdp 且 digest/node_count/path 必填") + elif source != "none" or any(value is not None for value in (digest, node_count, snapshot_path)): + _fail(path, "present=false 时 source=none 且 digest/node_count/path 必须为 null") + return BrowserSnapshot(present, source, digest, node_count, snapshot_path) + + def observation_from_dict(data: Any) -> Observation: path = "observation" obj = _as_obj(data, path) @@ -205,7 +227,7 @@ def observation_from_dict(data: Any) -> Observation: "last_receipt", "protocol_version", ], - [], + ["browser_snapshot"], path, ) budget_obj = _as_obj(obj["budget"], f"{path}.budget") @@ -227,6 +249,9 @@ def observation_from_dict(data: Any) -> Observation: screenshot=_screenshot_from_dict(obj["screenshot"], f"{path}.screenshot"), accessibility_tree=_a11y_from_dict(obj["accessibility_tree"], f"{path}.accessibility_tree"), last_receipt=last_receipt, + browser_snapshot=_browser_snapshot_from_dict( + obj.get("browser_snapshot"), f"{path}.browser_snapshot" + ), protocol_version=_as_enum(obj["protocol_version"], (PROTOCOL_VERSION,), f"{path}.protocol_version"), ) diff --git a/src/tandem/runtime/session.py b/src/tandem/runtime/session.py index f32f1a1..c95d18d 100644 --- a/src/tandem/runtime/session.py +++ b/src/tandem/runtime/session.py @@ -7,6 +7,7 @@ from __future__ import annotations +import json import time import urllib.error from pathlib import Path @@ -58,13 +59,19 @@ def act(self, observation: Observation) -> Action: if reason: return FailAction(reason=reason) screenshot = Path(observation.screenshot.path) - targets = ( + raw_targets = ( self.components.target_builder.build( tree, f"t{observation.step_idx}", 128, infer_button_labels=True, include_sliders=True ) if tree is not None else [] ) + targets = self.components.semantic_enricher( + raw_targets, + tree, + (observation.screenshot.width, observation.screenshot.height), + self._read_browser_snapshot(observation), + ) # 同一观察内最多重规划 NO_ACTION_LIMIT+3 次:完成被拒/无候选会递增 no_action 并重规划, # 与旧循环 continue 语义一致(旧版重取 obs,本帧无动作时内容等价)。 for _ in range(NO_ACTION_LIMIT + 3): @@ -149,6 +156,8 @@ def _plan_gui( space.target_heads["visual"] = visual if visual: space.controls.pop("visual_fallback", None) + else: + space.controls.pop("visual_fallback", None) prune_stalled(space, self.recent) ranking = None for service_retry in range(3): @@ -221,3 +230,13 @@ def _read_tree(self, observation: Observation) -> str | None: return Path(a11y.path).read_text(encoding="utf-8") except OSError: return None + + def _read_browser_snapshot(self, observation: Observation) -> dict | None: + reference = observation.browser_snapshot + if reference is None or not reference.present or not reference.path: + return None + try: + value = json.loads(Path(reference.path).read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + return value if isinstance(value, dict) else None diff --git a/src/tandem_eval/browser.py b/src/tandem_eval/browser.py new file mode 100644 index 0000000..358497c --- /dev/null +++ b/src/tandem_eval/browser.py @@ -0,0 +1,276 @@ +"""Read-only Chrome DevTools snapshots, captured inside the task VM.""" + +from __future__ import annotations + +import json +from typing import Any + +VM_SCRIPT = r''' +import base64 +import hashlib +import http.client +import json +import os +import socket +import urllib.request + +EXPRESSION = r""" +(() => { + const norm = value => String(value || '').replace(/\s+/g, ' ').trim(); + const visible = element => { + if (element.closest('[aria-hidden="true"],[inert]')) return false; + if (element.checkVisibility) return element.checkVisibility({checkOpacity:true, checkVisibilityCSS:true}); + const style = getComputedStyle(element); + return style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'; + }; + const name = (element, seen = new Set()) => { + if (!element || seen.has(element)) return ''; + seen.add(element); + const referenced = (element.getAttribute('aria-labelledby') || '').split(/\s+/) + .map(id => name(document.getElementById(id), seen)).filter(Boolean).join(' '); + return norm(referenced || element.getAttribute('aria-label') || + [...(element.labels || [])].map(label => name(label, seen)).filter(Boolean).join(' ') || + (['button','submit','reset'].includes(element.type) ? element.value : '') || + element.getAttribute('alt') || + (element.tagName === 'INPUT' ? '' : [...element.childNodes].map(node => + node.nodeType === 3 ? node.textContent : node.nodeType === 1 ? name(node, seen) : '').join(' ')) || + element.getAttribute('title') || element.getAttribute('placeholder') || element.getAttribute('name')); + }; + const roles = new Set(['button','link','checkbox','radio','switch','tab','menuitem','option', + 'gridcell','combobox','textbox','searchbox','spinbutton','slider']); + const selector = 'a[href],button,input,textarea,select,summary,[contenteditable="true"],' + + [...roles].map(role => '[role="' + role + '"]').join(','); + const secretAutocomplete = new RegExp( + '(^|\\s)(current-password|new-password|one-time-code|cc-number|' + + 'cc-csc|cc-exp|cc-exp-month|cc-exp-year)(\\s|$)'); + const roleOf = element => { + const explicit = element.getAttribute('role'); + if (roles.has(explicit)) return explicit; + if (element.tagName === 'BUTTON' || element.tagName === 'SUMMARY') return 'button'; + if (element.tagName === 'A') return 'link'; + if (element.tagName === 'SELECT') return 'combobox'; + if (element.tagName === 'TEXTAREA' || element.isContentEditable) return 'textbox'; + if (element.tagName === 'INPUT') { + if (['checkbox','radio'].includes(element.type)) return element.type; + if (['button','submit','reset','image'].includes(element.type)) return 'button'; + if (element.type === 'search') return 'searchbox'; + if (element.type === 'number') return 'spinbutton'; + if (['text','email','url','tel'].includes(element.type)) return 'textbox'; + } + return null; + }; + const roleNames = {button:'push_button', link:'link', checkbox:'check_box', radio:'radio_button', + switch:'switch', tab:'page_tab', menuitem:'menu_item', option:'option', gridcell:'table_cell', + combobox:'combo_box', textbox:'entry', searchbox:'searchbox', spinbutton:'spin_button', slider:'slider'}; + const originX = window.screenX + Math.max(0, (window.outerWidth - window.innerWidth) / 2); + const originY = window.screenY + Math.max(0, window.outerHeight - window.innerHeight); + const textCandidates = []; + for (const element of document.querySelectorAll('label,th,td,legend,p,span,h1,h2,h3,h4,h5,h6')) { + if (element.querySelector('input,textarea,select,button') || !visible(element)) continue; + const text = norm(element.innerText); + const rect = element.getBoundingClientRect(); + if (text && text.length <= 100 && rect.width > 0 && rect.height > 0) { + textCandidates.push({text, left:rect.left, right:rect.right, top:rect.top, bottom:rect.bottom, + cx:rect.left + rect.width / 2, cy:rect.top + rect.height / 2}); + } + if (textCandidates.length >= 600) break; + } + const hintFor = rect => { + const cx = rect.left + rect.width / 2, cy = rect.top + rect.height / 2; + let best = '', score = Infinity; + for (const item of textCandidates) { + const left = item.right <= rect.left + 4 && Math.abs(item.cy - cy) <= Math.max(14, rect.height); + const above = item.bottom <= rect.top + 4 && item.right >= rect.left && item.left <= rect.right; + if (!left && !above) continue; + const candidate = left ? rect.left - item.right + Math.abs(item.cy - cy) : + 1000 + rect.top - item.bottom + Math.abs(item.cx - cx) / 4; + if (candidate < score) { best = item.text; score = candidate; } + } + return best; + }; + const contextText = element => { + if (!element) return ''; + const clone = element.cloneNode(true); + clone.querySelectorAll('input,textarea,select,[contenteditable="true"]').forEach(node => node.remove()); + return norm(clone.innerText).slice(0, 240); + }; + const elements = []; + let rawCount = 0; + for (const element of document.querySelectorAll(selector)) { + rawCount++; + if (!visible(element) || element.disabled || element.getAttribute('aria-disabled') === 'true') continue; + if (element.type === 'password' || element.type === 'hidden' || element.type === 'file' || + secretAutocomplete.test(element.getAttribute('autocomplete') || '')) continue; + const rect = element.getBoundingClientRect(); + const cx = rect.left + rect.width / 2, cy = rect.top + rect.height / 2; + if (rect.width < 2 || rect.height < 2 || cx < 0 || cy < 0 || + cx >= innerWidth || cy >= innerHeight) continue; + const domRole = roleOf(element); + if (!domRole) continue; + const hint = hintFor(rect); + const label = name(element) || hint || domRole; + const editable = !element.readOnly && element.getAttribute('aria-readonly') !== 'true' && + (['textbox','searchbox','spinbutton'].includes(domRole) || + (domRole === 'combobox' && ['INPUT','TEXTAREA'].includes(element.tagName))); + const contextElement = element.closest('form,dialog,[role="dialog"],fieldset,li,tr,[role="row"]'); + const item = { + index: elements.length + 1, + role: roleNames[domRole] || domRole, + name: label, + description: norm(element.getAttribute('aria-description') || element.getAttribute('title') || + element.getAttribute('placeholder')), + value: 'value' in element ? String(element.value).slice(0, 200) : '', + x: originX + rect.left, y: originY + rect.top, width: rect.width, height: rect.height, + scope: 'web_content', label_source: name(element) ? 'dom' : 'dom_nearby', + nearby_text: contextText(contextElement), + operations: editable ? ['click','type_text'] : ['click'], + source: 'dom' + }; + for (const state of ['checked','selected','expanded']) { + const value = element.getAttribute('aria-' + state); + if (value !== null) item[state] = value === 'true'; + } + if (['checkbox','radio'].includes(element.type)) item.checked = Boolean(element.checked); + elements.push(item); + if (elements.length >= 250) break; + } + return {url:location.href, title:document.title, visibility:document.visibilityState, elements, + omitted:Math.max(0, rawCount - elements.length), + viewport:{width:innerWidth,height:innerHeight,screen_x:originX,screen_y:originY,dpr:devicePixelRatio}}; +})() +""" + + +def read_exact(sock, size): + chunks = [] + while size: + chunk = sock.recv(size) + if not chunk: + raise OSError('CDP websocket closed') + chunks.append(chunk) + size -= len(chunk) + return b''.join(chunks) + + +def send_frame(sock, payload, opcode=1): + data = payload.encode() if isinstance(payload, str) else payload + mask = os.urandom(4) + size = len(data) + header = bytes([0x80 | opcode, 0x80 | (size if size < 126 else 126 if size < 65536 else 127)]) + if size >= 65536: + header += size.to_bytes(8, 'big') + elif size >= 126: + header += size.to_bytes(2, 'big') + masked = bytes(value ^ mask[index % 4] for index, value in enumerate(data)) + sock.sendall(header + mask + masked) + + +def recv_frame(sock): + first, second = read_exact(sock, 2) + size = second & 127 + if size == 126: + size = int.from_bytes(read_exact(sock, 2), 'big') + elif size == 127: + size = int.from_bytes(read_exact(sock, 8), 'big') + mask = read_exact(sock, 4) if second & 128 else None + payload = read_exact(sock, size) + if mask: + payload = bytes(value ^ mask[index % 4] for index, value in enumerate(payload)) + return bool(first & 128), first & 15, payload + + +def evaluate(port, page): + ws = page['webSocketDebuggerUrl'] + assert ws.startswith('ws://') + rest = ws[5:] + hostport, path = rest.split('/', 1) + host, port_text = hostport.rsplit(':', 1) + sock = socket.create_connection((host, int(port_text)), timeout=5) + sock.settimeout(5) + try: + key = base64.b64encode(os.urandom(16)).decode() + request = ('GET /' + path + ' HTTP/1.1\r\nHost: ' + hostport + + '\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: ' + key + + '\r\nSec-WebSocket-Version: 13\r\n\r\n') + sock.sendall(request.encode()) + response = b'' + while b'\r\n\r\n' not in response: + response += sock.recv(4096) + if b' 101 ' not in response.split(b'\r\n', 1)[0]: + raise OSError('CDP websocket handshake failed') + send_frame(sock, json.dumps({'id':1, 'method':'Runtime.evaluate', + 'params':{'expression':EXPRESSION, 'returnByValue':True, 'awaitPromise':True}})) + chunks = None + for _ in range(100): + finished, opcode, payload = recv_frame(sock) + if opcode == 9: + send_frame(sock, payload, 10) + continue + if opcode == 1: + chunks = [payload] + elif opcode == 0 and chunks is not None: + chunks.append(payload) + else: + continue + if not finished: + continue + message = json.loads(b''.join(chunks)) + chunks = None + if message.get('id') != 1: + continue + if 'error' in message or 'exceptionDetails' in message.get('result', {}): + raise OSError('CDP Runtime.evaluate failed') + return message['result']['result']['value'] + raise OSError('CDP response limit reached') + finally: + sock.close() + + +def main(): + errors = [] + for port in (1337, 9222): + try: + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + with opener.open(f'http://127.0.0.1:{port}/json/list', timeout=3) as response: + pages = json.load(response) + candidates = [item for item in pages if item.get('type') == 'page'] + snapshot = None + for page in candidates: + candidate = evaluate(port, page) + if snapshot is None: + snapshot = candidate + if candidate.get('visibility') == 'visible': + snapshot = candidate + break + if snapshot is None: + raise OSError('no CDP page target') + print(json.dumps({'snapshot':snapshot, 'port':port}, ensure_ascii=False)) + break + except Exception as error: + errors.append(f'{port}:{type(error).__name__}:{str(error)[:160]}') + else: + print(json.dumps({'error':' | '.join(errors)}, ensure_ascii=False)) + + +if __name__ == '__main__': + main() +''' + + +def capture_browser_snapshot(controller) -> tuple[dict[str, Any] | None, dict[str, Any]]: + """Capture the active page without changing it; failures degrade to a11y-only perception.""" + try: + result = controller.execute_python_command(VM_SCRIPT) or {} + payload = json.loads(result.get("output") or "{}") + except (TypeError, ValueError, AttributeError) as error: + return None, {"available": False, "error": f"{type(error).__name__}: {error}"[:240]} + snapshot = payload.get("snapshot") + if not isinstance(snapshot, dict) or not isinstance(snapshot.get("elements"), list): + return None, {"available": False, "error": str(payload.get("error") or "invalid CDP response")[:240]} + return snapshot, { + "available": True, + "port": payload.get("port"), + "url": snapshot.get("url"), + "element_count": len(snapshot["elements"]), + "omitted": snapshot.get("omitted", 0), + } diff --git a/src/tandem_eval/entry.py b/src/tandem_eval/entry.py index 19547e5..76ba5c9 100644 --- a/src/tandem_eval/entry.py +++ b/src/tandem_eval/entry.py @@ -150,12 +150,14 @@ def mock_components(*, stage_mode=False) -> Components: from tandem.gui.candidates import ActionCandidateBuilder from tandem.gui.compile import gui_action from tandem.gui.proposals import focused_candidates + from tandem.gui.semantics import enrich_targets from tandem.gui.targets import A11yTargetBuilder return Components( planner=_ScriptedPlanner(stage_mode=stage_mode), verifier=_ScriptedVerifier(), requirements=lambda instruction: [], selector=_FirstCandidateSelector(), target_builder=A11yTargetBuilder(), candidate_builder=ActionCandidateBuilder(), + semantic_enricher=enrich_targets, focus_candidates=focused_candidates, grounder=None, compile_gui=gui_action, ) diff --git a/src/tandem_eval/osworld.py b/src/tandem_eval/osworld.py index 3390a99..f8d3905 100644 --- a/src/tandem_eval/osworld.py +++ b/src/tandem_eval/osworld.py @@ -23,6 +23,7 @@ from tandem.protocol.models import ( AccessibilityTree, Action, + BrowserSnapshot, Budget, CliAction, FailAction, @@ -34,6 +35,8 @@ WaitAction, ) +from .browser import capture_browser_snapshot + RECEIPT_TAIL = 12000 @@ -125,6 +128,8 @@ def _bootstrap_runtime(self) -> None: self._step_seq = 0 self.last_screenshot: bytes = b"" self.last_a11y: str = "" + self.last_browser_snapshot: dict | None = None + self._cdp_reported = False @property def name(self) -> str: @@ -188,6 +193,18 @@ def _observation(self, task: dict, step_idx: int, budget: Budget, digest = hashlib.sha256(self.last_a11y.encode()).hexdigest() node_count = self.last_a11y.count("<") width, height = png_size(self.last_screenshot) + browser_snapshot = None + if self.last_browser_snapshot is not None: + serialized = json.dumps(self.last_browser_snapshot, ensure_ascii=False) + browser_path = stage / f"step_{step_idx:02d}.dom.json" + browser_path.write_text(serialized, encoding="utf-8") + browser_snapshot = BrowserSnapshot( + present=True, + source="cdp", + digest=hashlib.sha256(serialized.encode()).hexdigest(), + node_count=len(self.last_browser_snapshot.get("elements", [])), + path=str(browser_path), + ) return Observation( task_id=str(task["id"]), instruction=str(task["instruction"]), @@ -199,12 +216,18 @@ def _observation(self, task: dict, step_idx: int, budget: Budget, present=present, source="vm" if present else "none", digest=digest, node_count=node_count, path=a11y_path), last_receipt=receipt, + browser_snapshot=browser_snapshot, ) def _capture(self) -> None: obs = self.env._get_obs() self.last_screenshot = obs["screenshot"] self.last_a11y = obs.get("accessibility_tree") or "" + self.last_browser_snapshot, diagnostic = capture_browser_snapshot(self.env.controller) + if not self._cdp_reported: + self._cdp_reported = True + public = {key: value for key, value in diagnostic.items() if key != "url"} + print(json.dumps({"event": "cdp_snapshot", **public}, ensure_ascii=False), flush=True) # ------------------------------------------------------------------ 协议 def _seed_setup_cache(self, task_id: str) -> None: @@ -231,6 +254,7 @@ def reset(self, task: dict) -> Observation: require_a11y_tree=True, cache_dir=str(self.run_root / "evaluation_cache"), ) self.env.reset(task_config=task) + self._cdp_reported = False time.sleep(self.startup_wait_seconds) self._capture() self._step_seq = 0 diff --git a/src/tandem_eval/recorder.py b/src/tandem_eval/recorder.py index 62fe82b..fe26c6c 100644 --- a/src/tandem_eval/recorder.py +++ b/src/tandem_eval/recorder.py @@ -1,7 +1,7 @@ """结果目录契约执行者(契约文档:docs/RESULT_DIR_CONTRACT.md)。 episode 目录自包含:task.json 副本、traj.jsonl、llm_calls.jsonl(全量请求/响应,脱敏)、 -screenshots/、a11y/、events.jsonl、deliverables/、evaluation.json、result.txt、 +screenshots/、a11y/、browser/、events.jsonl、deliverables/、evaluation.json、result.txt、 episode.json、timing.json。run 级:summary.json、heartbeat.json、code_version.json。 """ @@ -10,6 +10,7 @@ import hashlib import json import re +import shutil import time import traceback from pathlib import Path @@ -64,6 +65,13 @@ def __init__(self, episode_dir: Path, task: dict, *, a11y_full: bool = False): # ------------------------------------------------------------------ 逐步记录 def traj(self, observation: Observation, action: Action, receipt: Receipt | None) -> None: + browser_snapshot = observation.browser_snapshot + browser_record = browser_snapshot.to_dict() if browser_snapshot else None + if browser_snapshot and browser_snapshot.present and browser_snapshot.path: + target = self.dir / "browser" / f"step_{observation.step_idx - 1:02d}.json" + target.parent.mkdir(exist_ok=True) + shutil.copyfile(browser_snapshot.path, target) + browser_record["path"] = str(target) record = { "step_idx": observation.step_idx, "wall_time": round(time.time(), 3), @@ -71,6 +79,7 @@ def traj(self, observation: Observation, action: Action, receipt: Receipt | None "observation": { "screenshot": observation.screenshot.to_dict(), "accessibility_tree": observation.accessibility_tree.to_dict(), + "browser_snapshot": browser_record, "budget": observation.budget.to_dict(), "last_receipt": receipt.to_dict() if receipt else None, }, diff --git a/tests/eval/test_browser.py b/tests/eval/test_browser.py new file mode 100644 index 0000000..2bb9219 --- /dev/null +++ b/tests/eval/test_browser.py @@ -0,0 +1,77 @@ +"""CDP response handling stays read-only and degrades without breaking observation.""" + +import json + +from tandem_eval.browser import VM_SCRIPT, capture_browser_snapshot + + +class Controller: + def __init__(self, payload): + self.payload = payload + self.commands = [] + + def execute_python_command(self, command): + self.commands.append(command) + return {"returncode": 0, "output": json.dumps(self.payload)} + + +def test_cdp_snapshot_response_and_public_diagnostic(): + snapshot = {"url": "https://example.test", "elements": [{"name": "Save"}], "omitted": 0} + controller = Controller({"snapshot": snapshot, "port": 1337}) + value, diagnostic = capture_browser_snapshot(controller) + assert value == snapshot + assert diagnostic == { + "available": True, "port": 1337, "url": "https://example.test", + "element_count": 1, "omitted": 0, + } + assert controller.commands == [VM_SCRIPT] + + +def test_cdp_failure_returns_a11y_fallback_diagnostic(): + value, diagnostic = capture_browser_snapshot(Controller({"error": "connection refused"})) + assert value is None + assert diagnostic["available"] is False and "connection refused" in diagnostic["error"] + + +def test_vm_websocket_frame_parser_executes_with_stdlib_only(): + namespace = {"__name__": "cdp_test"} + exec(VM_SCRIPT, namespace) + + class Socket: + def __init__(self): + self.data = b"\x81\x02{}" + + def recv(self, size): + value, self.data = self.data[:size], self.data[size:] + return value + + assert namespace["recv_frame"](Socket()) == (True, 1, b"{}") + + +def test_vm_evaluate_reads_fragmented_cdp_response(): + namespace = {"__name__": "cdp_test"} + exec(VM_SCRIPT, namespace) + + class Socket: + def settimeout(self, timeout): + pass + + def sendall(self, data): + pass + + def recv(self, size): + return b"HTTP/1.1 101 Switching Protocols\r\n\r\n" + + def close(self): + pass + + socket = Socket() + namespace["socket"].create_connection = lambda address, timeout: socket + namespace["send_frame"] = lambda *args: None + frames = iter([ + (False, 1, b'{"id":1,"result":'), + (True, 0, b'{"result":{"value":{"elements":[]}}}}'), + ]) + namespace["recv_frame"] = lambda sock: next(frames) + page = {"webSocketDebuggerUrl": "ws://127.0.0.1:1337/devtools/page/1"} + assert namespace["evaluate"](1337, page) == {"elements": []} diff --git a/tests/eval/test_mock_chain.py b/tests/eval/test_mock_chain.py index 9a4fce9..5391883 100644 --- a/tests/eval/test_mock_chain.py +++ b/tests/eval/test_mock_chain.py @@ -58,7 +58,7 @@ def test_full_mock_chain_produces_renderable_result_dir(tmp_path, mode): assert (episode_dir / "screenshots" / "step_00.png").is_file() traj = [json.loads(line) for line in (episode_dir / "traj.jsonl").read_text().splitlines()] - assert traj and traj[0]["protocol_version"] == "0.1" + assert traj and traj[0]["protocol_version"] == "0.2" assert traj[0]["action"]["kind"] == "gui" episode = json.loads((episode_dir / "episode.json").read_text()) assert episode["stop_reason"] == "done" and episode["score"] == 1.0 diff --git a/tests/eval/test_osworld_env.py b/tests/eval/test_osworld_env.py index ec0806f..10fe121 100644 --- a/tests/eval/test_osworld_env.py +++ b/tests/eval/test_osworld_env.py @@ -169,6 +169,20 @@ def test_reset_execute_score_lifecycle(tmp_path, env): assert fake.closed +def test_browser_snapshot_is_a_hashed_observation_file(tmp_path, env, monkeypatch): + monkeypatch.setattr( + "tandem_eval.osworld.capture_browser_snapshot", + lambda controller: ({"elements": [{"name": "Save"}], "url": "https://example.test"}, + {"available": True}), + ) + task = _task_file(tmp_path) + observation = env.reset(task) + reference = observation.browser_snapshot + assert reference is not None and reference.present and reference.source == "cdp" + assert reference.node_count == 1 and len(reference.digest) == 64 + assert json.loads(Path(reference.path).read_text())["elements"][0]["name"] == "Save" + + def test_ack_discipline_requires_receipt(tmp_path, env): task = _task_file(tmp_path) env.reset(task) diff --git a/tests/eval/test_recorder_contract.py b/tests/eval/test_recorder_contract.py index b66fa66..3efe443 100644 --- a/tests/eval/test_recorder_contract.py +++ b/tests/eval/test_recorder_contract.py @@ -3,13 +3,16 @@ from __future__ import annotations import json +from dataclasses import replace +from tandem.protocol.models import BrowserSnapshot, WaitAction from tandem_eval.recorder import ( EpisodeRecorder, write_code_version, write_heartbeat, write_summary, ) +from tests.harness.conftest import make_observation def test_episode_recorder_writes_contract_files(tmp_path): @@ -37,6 +40,24 @@ def test_episode_recorder_writes_contract_files(tmp_path): assert episode["status"] == "scored" and episode["steps"] == 3 +def test_traj_preserves_browser_snapshot_inside_episode(tmp_path): + source = tmp_path / "obs_stage" / "step_01.dom.json" + source.parent.mkdir() + source.write_text('{"elements":[{"name":"Save"}]}', encoding="utf-8") + observation = replace( + make_observation(tmp_path, 1), + browser_snapshot=BrowserSnapshot(True, "cdp", "d" * 64, 1, str(source)), + ) + recorder = EpisodeRecorder(tmp_path / "ep", {"id": "t1"}) + recorder.traj(observation, WaitAction(), None) + recorder.close() + record = json.loads((tmp_path / "ep" / "traj.jsonl").read_text()) + reference = record["observation"]["browser_snapshot"] + assert reference["present"] is True and reference["source"] == "cdp" + assert (tmp_path / "ep" / "browser" / "step_00.json").is_file() + assert reference["path"] == str(tmp_path / "ep" / "browser" / "step_00.json") + + def test_llm_calls_redacts_bearer_secrets(tmp_path): recorder = EpisodeRecorder(tmp_path / "ep", {"id": "t1"}) recorder.record_llm_call( @@ -75,7 +96,7 @@ def test_heartbeat_and_code_version(tmp_path): assert json.loads((tmp_path / "heartbeat.json").read_text())["current_task"] == "task-2" write_code_version(tmp_path, {"git_describe": "abc123", "resources": {"schema_version": "v1"}}) info = json.loads((tmp_path / "code_version.json").read_text()) - assert info["git_describe"] == "abc123" and info["protocol_version"] == "0.1" + assert info["git_describe"] == "abc123" and info["protocol_version"] == "0.2" def test_run_spec_masks_credentials_but_keeps_model_identity(): diff --git a/tests/harness/conftest.py b/tests/harness/conftest.py index 33fd4d8..f7317ab 100644 --- a/tests/harness/conftest.py +++ b/tests/harness/conftest.py @@ -12,6 +12,7 @@ from tandem.gui.candidates import ActionCandidateBuilder from tandem.gui.compile import gui_action from tandem.gui.proposals import focused_candidates +from tandem.gui.semantics import enrich_targets from tandem.gui.targets import A11yTargetBuilder from tandem.models.ranking import Ranking from tandem.protocol.models import ( @@ -137,7 +138,11 @@ class FixedSelector: def rank(self, *, stage_goal, intent, active_window, targets, action_space, recent_actions, screenshot=None): - self.calls.append({"intent": intent, "n_candidates": len(action_space.candidates())}) + self.calls.append({ + "intent": intent, + "n_candidates": len(action_space.candidates()), + "operations": [candidate.operation for candidate in action_space.candidates()], + }) candidates = action_space.candidates() winner = candidates[0] if candidates else None return Ranking( @@ -171,6 +176,7 @@ def make_components(planner, verifier, selector) -> Components: requirements=lambda instruction: [], selector=selector, target_builder=A11yTargetBuilder(), + semantic_enricher=enrich_targets, candidate_builder=ActionCandidateBuilder(), focus_candidates=focused_candidates, grounder=None, diff --git a/tests/harness/test_semantics.py b/tests/harness/test_semantics.py new file mode 100644 index 0000000..a7577a3 --- /dev/null +++ b/tests/harness/test_semantics.py @@ -0,0 +1,128 @@ +"""Shared semantic perception, scoped candidates, and compact JEV requests.""" + +import json + +from tandem.gui.candidates import ActionCandidateBuilder +from tandem.gui.proposals import focused_candidates +from tandem.gui.semantics import enrich_targets +from tandem.gui.targets import A11yTargetBuilder +from tandem.models.ranking import build_choice_requests + +CHROME_XML = """ + + + + + """ + + +def _targets(snapshot=None): + raw = A11yTargetBuilder().build(CHROME_XML, "t8", 128, infer_button_labels=True) + return enrich_targets(raw, CHROME_XML, (1920, 1080), snapshot) + + +def _named(targets, name): + return next(item for item in targets if item.name == name) + + +def _space(targets): + return ActionCandidateBuilder().build(targets, 96, action_inputs=[], context="expense") + + +def test_a11y_recovers_nearby_labels_and_separates_browser_chrome(): + targets = _targets() + date = _named(targets, "Receipt Date") + amount = _named(targets, "Receipt Amount") + assert date.label_source == "a11y_nearby" + assert date.value == "500.00" and "500.00" not in date.name + assert date.scope == amount.scope == "web_content" + assert _named(targets, "Minimise").scope == "browser_chrome" + + +def test_dom_enriches_matching_a11y_target_and_adds_dom_only_controls(): + date = _named(_targets(), "Receipt Date") + snapshot = { + "viewport": {"screen_y": 112}, + "elements": [ + { + "index": 1, "role": "entry", "name": "Receipt Date", + "description": "Receipt date", "value": "11/12/2025", + "x": date.x, "y": date.y, "width": date.width, "height": date.height, + "operations": ["click", "type_text"], "label_source": "dom", + }, + { + "index": 2, "role": "push_button", "name": "Upload receipt", + "x": 900, "y": 400, "width": 120, "height": 30, + "operations": ["click"], "label_source": "dom", + }, + ], + } + targets = _targets(snapshot) + merged = _named(targets, "Receipt Date") + assert merged.id == date.id and merged.source == "a11y+dom" + assert merged.value == "11/12/2025" and merged.operations == ("click", "type_text") + assert _named(targets, "Upload receipt").source == "dom" + + +def test_web_proposal_does_not_reintroduce_browser_shell_candidates(): + targets = _targets() + inbox = _named(targets, "Inbox 2") + proposal = {"operation": "click", "target_id": inbox.id, + "target_name": inbox.name, "target_role": inbox.role} + space = focused_candidates(_space(targets), targets, CHROME_XML, [proposal], None) + names = {target.name for target in targets if target.id in {c.target_id for c in space.candidates()}} + assert "Inbox 2" in names and "Save" in names + assert "Minimise" not in names and "New Tab" not in names + + +def test_shell_proposal_does_not_reintroduce_web_candidates(): + targets = _targets() + tab = _named(targets, "New Tab") + proposal = {"operation": "click", "target_id": tab.id, + "target_name": tab.name, "target_role": tab.role} + space = focused_candidates(_space(targets), targets, CHROME_XML, [proposal], None) + names = {target.name for target in targets if target.id in {c.target_id for c in space.candidates()}} + assert "New Tab" in names and "Minimise" in names + assert "Inbox 2" not in names and "Save" not in names + + +def test_focused_typing_rejects_a_different_named_field(): + targets = _targets() + proposal = {"operation": "type_text", "target_name": "Receipt Amount", + "target_role": "entry", "text": "500.00", "input_mode": "replace"} + space = focused_candidates(_space(targets), targets, CHROME_XML, [proposal], None) + assert not [candidate for candidate in space.candidates() if candidate.operation == "type_text"] + proposal["target_name"] = "Receipt Date" + space = focused_candidates(_space(targets), targets, CHROME_XML, [proposal], None) + assert [candidate for candidate in space.candidates() if candidate.operation == "type_text"] + + +def test_jev_request_has_semantics_but_not_full_verification_history(): + targets = _targets() + inbox = _named(targets, "Inbox 2") + proposal = {"operation": "click", "target_id": inbox.id, + "target_name": inbox.name, "target_role": inbox.role} + space = focused_candidates(_space(targets), targets, CHROME_XML, [proposal], None) + recent = [{"operation": "click", "target": "Save", "outcome": "achieved", + "verification": {"evidence": "large"}} + ] + operation, actions = build_choice_requests( + stage_goal="Expense", intent="Open inbox", active_window="", targets=targets, + action_space=space, recent_actions=recent, + ) + state = json.loads(operation["state"]) + assert state["recent_semantic_actions"] == [ + {"operation": "click", "target": "Save", "outcome": "achieved"} + ] + assert "verification" not in operation["state"] + assert "Scope: web_content" in actions["click"]["options"][0] diff --git a/tests/harness/test_session.py b/tests/harness/test_session.py index 2576ac3..b73f226 100644 --- a/tests/harness/test_session.py +++ b/tests/harness/test_session.py @@ -245,6 +245,15 @@ def test_wait_candidate_emits_wait_action(tmp_path): assert isinstance(action, WaitAction) +def test_unavailable_visual_fallback_is_not_offered_to_selector(tmp_path): + selector = FixedSelector() + session, _, _ = _session([_gui_plan()], [], selector=selector) + session.begin("整理", []) + action = session.act(make_observation(tmp_path, 0)) + assert isinstance(action, GuiAction) + assert "visual_fallback" not in selector.calls[0]["operations"] + + def test_cli_timeout_clamped_to_protocol_limit(): with pytest.raises(ValueError, match="cli_timeout_seconds"): SessionConfig(cli_timeout_seconds=200) diff --git a/tests/harness/test_targets.py b/tests/harness/test_targets.py index f395058..6e31d22 100644 --- a/tests/harness/test_targets.py +++ b/tests/harness/test_targets.py @@ -2,6 +2,7 @@ from __future__ import annotations +from tandem.gui.semantics import enrich_targets from tandem.gui.targets import A11yTargetBuilder from .conftest import A11Y_XML @@ -24,7 +25,9 @@ def test_button_label_inferred_from_single_label_child(): def test_focused_unnamed_editable_gets_synthetic_name(): - targets = A11yTargetBuilder().build(A11Y_XML, "snap", 128) + targets = enrich_targets( + A11yTargetBuilder().build(A11Y_XML, "snap", 128), A11Y_XML, (1920, 1080) + ) names = {t.name for t in targets} assert any(name.startswith("Focused textbox") for name in names) diff --git a/tests/protocol/test_golden_samples.py b/tests/protocol/test_golden_samples.py index 0002650..c4a21b5 100644 --- a/tests/protocol/test_golden_samples.py +++ b/tests/protocol/test_golden_samples.py @@ -33,6 +33,7 @@ "action_wait", "observation_no_a11y", "observation_osw1_step0", + "observation_osw2_cdp", "observation_osw2_jev", "receipt_cli", "receipt_cli_error", diff --git a/tests/protocol/test_validate_rejects.py b/tests/protocol/test_validate_rejects.py index 1f7f182..762d1f7 100644 --- a/tests/protocol/test_validate_rejects.py +++ b/tests/protocol/test_validate_rejects.py @@ -95,7 +95,7 @@ def _observation(**overrides): ("receipt-stdout-too-long", receipt_from_dict, {**VALID_RECEIPT, "stdout": "x" * 12001}, "12000"), ("receipt-stdout-int", receipt_from_dict, {**VALID_RECEIPT, "stdout": 123}, "必须是 string"), # --- Observation --- - ("obs-bad-version", observation_from_dict, _observation(protocol_version="0.2"), "protocol_version"), + ("obs-bad-version", observation_from_dict, _observation(protocol_version="0.1"), "protocol_version"), ("obs-negative-step", observation_from_dict, _observation(step_idx=-1), "必须 >="), ("obs-unknown-top-field", observation_from_dict, _observation(extra=1), "未知字段"), ( @@ -173,3 +173,17 @@ def test_rejects_a11y_absent_with_leftovers(): ) with pytest.raises(ProtocolValidationError, match="必须为 null"): observation_from_dict(bad) + + +def test_rejects_cdp_present_without_required_references(): + bad = _observation(browser_snapshot={"present": True, "source": "cdp"}) + with pytest.raises(ProtocolValidationError, match="digest/node_count/path"): + observation_from_dict(bad) + + +def test_rejects_cdp_absent_with_leftovers(): + bad = _observation( + browser_snapshot={"present": False, "source": "none", "path": "obs_stage/t/dom.json"} + ) + with pytest.raises(ProtocolValidationError, match="必须为 null"): + observation_from_dict(bad) diff --git a/tests/test_package.py b/tests/test_package.py index e8fbc12..206f6a4 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -9,7 +9,7 @@ def test_version(): def test_protocol_exports(): - assert PROTOCOL_VERSION == "0.1" + assert PROTOCOL_VERSION == "0.2" assert callable(action_from_dict) assert callable(observation_from_dict) assert callable(receipt_from_dict) From 2404ae4bfb33f2311b0bebc3f3e2c2f790daeee8 Mon Sep 17 00:00:00 2001 From: tandem-demo Date: Thu, 29 Oct 2026 14:43:46 +0800 Subject: [PATCH 02/11] fix: honor namespaced active state in DOM merging --- src/tandem/gui/semantics.py | 11 ++++++++--- tests/harness/test_semantics.py | 9 +++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/tandem/gui/semantics.py b/src/tandem/gui/semantics.py index 379ebd9..bffc734 100644 --- a/src/tandem/gui/semantics.py +++ b/src/tandem/gui/semantics.py @@ -31,10 +31,15 @@ def _a11y_context(root: ET.Element | None) -> tuple[list[dict], bool]: return [], True labels = [] chrome_active = False - for app in root: - app_active = app.attrib.get("active") == "true" or any( - child.attrib.get("active") == "true" for child in app + + def active(element: ET.Element) -> bool: + return any( + key.rsplit("}", 1)[-1] == "active" and value == "true" + for key, value in element.attrib.items() ) + + for app in root: + app_active = active(app) or any(active(child) for child in app) chrome_active = chrome_active or ( app_active and "chrome" in app.attrib.get("name", "").lower() ) diff --git a/tests/harness/test_semantics.py b/tests/harness/test_semantics.py index a7577a3..bbf3c5e 100644 --- a/tests/harness/test_semantics.py +++ b/tests/harness/test_semantics.py @@ -10,16 +10,17 @@ CHROME_XML = """ - + xmlns:attr="https://accessibility.ubuntu.example.org/ns/attributes" + xmlns:st="https://accessibility.ubuntu.example.org/ns/state"> +