From cc3aa50977d5976e75179be3143bb24a46fa8a1a Mon Sep 17 00:00:00 2001 From: xxxyxun Date: Sat, 31 Oct 2026 18:17:50 +0800 Subject: [PATCH 1/4] feat(memory): integrate goal execution and task memory --- README.md | 2 + docs/RESULT_DIR_CONTRACT.md | 16 +- docs/STAGE_EXECUTION.md | 64 ++++ docs/VALIDATION_CN.md | 90 +++++ infra/config.example.json | 7 +- scripts/run_experiment.sh | 18 +- src/tandem/components.py | 21 +- src/tandem/config.py | 11 +- src/tandem/gui/compile.py | 10 +- src/tandem/gui/goal_candidates.py | 187 ++++++++++ src/tandem/gui/grounding.py | 66 ++++ src/tandem/gui/targets.py | 75 +++- src/tandem/memory/__init__.py | 1 + src/tandem/memory/context.py | 59 ++++ src/tandem/memory/state.py | 229 ++++++++++++ src/tandem/models/jev.py | 15 +- src/tandem/models/slow.py | 69 +++- src/tandem/planning/goal_planner.py | 258 ++++++++++++++ src/tandem/runtime/goal_stage.py | 401 +++++++++++++++++++++ src/tandem/runtime/session.py | 7 + src/tandem/verification/goal_verifier.py | 247 +++++++++++++ src/tandem_eval/__main__.py | 2 +- src/tandem_eval/config_schema.py | 4 +- src/tandem_eval/entry.py | 28 +- src/tandem_eval/env.py | 12 +- src/tandem_eval/osworld.py | 34 +- src/tandem_eval/recorder.py | 51 ++- src/tandem_eval/rjob_spec.py | 8 +- src/tandem_eval/runner.py | 38 +- tests/eval/test_experiment_script.py | 10 + tests/eval/test_mock_chain.py | 11 +- tests/eval/test_osworld_env.py | 24 ++ tests/harness/conftest.py | 3 +- tests/harness/test_goal_execution.py | 201 +++++++++++ tests/harness/test_goal_integration.py | 114 ++++++ tests/harness/test_goal_memory.py | 265 ++++++++++++++ tests/harness/test_goal_progress.py | 161 +++++++++ tests/harness/test_goal_stage.py | 428 +++++++++++++++++++++++ 38 files changed, 3198 insertions(+), 49 deletions(-) create mode 100644 docs/VALIDATION_CN.md create mode 100644 src/tandem/gui/goal_candidates.py create mode 100644 src/tandem/gui/grounding.py create mode 100644 src/tandem/memory/__init__.py create mode 100644 src/tandem/memory/context.py create mode 100644 src/tandem/memory/state.py create mode 100644 src/tandem/planning/goal_planner.py create mode 100644 src/tandem/runtime/goal_stage.py create mode 100644 src/tandem/verification/goal_verifier.py create mode 100644 tests/harness/test_goal_execution.py create mode 100644 tests/harness/test_goal_integration.py create mode 100644 tests/harness/test_goal_memory.py create mode 100644 tests/harness/test_goal_progress.py create mode 100644 tests/harness/test_goal_stage.py diff --git a/README.md b/README.md index 740d3a0..abf90a9 100644 --- a/README.md +++ b/README.md @@ -336,4 +336,6 @@ python -m http.server 8899 --directory demo/site # 本地预览 默认保持逐步规划;启用 `--execution-mode stage` 后,由慢模型规划阶段、快模型连续选择动作, 规则验证优先,仅在必要时调用模型验证或重规划。 +`--execution-mode goal` 可启用带任务记忆的局部连续执行,阶段结束再独立 Verify; +保留 A11y/CDP 语义融合及 JEV 服务恢复,默认 step/stage 行为不变。 配置、执行边界、开发入口和验收方法见 [阶段执行说明](docs/STAGE_EXECUTION.md)。 diff --git a/docs/RESULT_DIR_CONTRACT.md b/docs/RESULT_DIR_CONTRACT.md index 6867654..5cda70c 100644 --- a/docs/RESULT_DIR_CONTRACT.md +++ b/docs/RESULT_DIR_CONTRACT.md @@ -39,6 +39,12 @@ OSWorld 2.1 显式配置替代 judge/用户模拟器时,`code_version.json.res | `result.txt` | 分数(`score\n`) | 同上 | | `episode.json` | `{task_id, harness, stop_reason, steps, status(scored\|failed_unscored), score, exception_type, error?, error_traceback?}` | episode 终态(必写)。`error`/`error_traceback` 仅异常题有:异常消息与栈尾几行(脱敏)——只记类型事后没法定位 | | `timing.json` | `{wall_seconds, status}` | episode 终态 | +| `task_memory.json`(goal) | 最新任务记忆:要求、事实、资料覆盖、交付单元、成果、失败与当前阶段 | goal 事件排水及终态 | +| `evidence/`(goal) | 阶段验证引用的截图/CLI 全文正本;临时目录清理后仍可读取 | 阶段验收/记忆落盘 | + +`timing.json` 另外提供 `environment_seconds`、`model_usage_by_role`、`fast_model`,分别记录环境 +分段时间、逐角色请求时间/token、排序次数/耗时/图像使用次数。缺失 usage 不补零冒充已测值, +另记 `calls_missing_usage`。原字段及 `browser/` 正本规则保持不变。 ## 步号语义(demo 侧对齐依据) @@ -63,7 +69,9 @@ OSWorld 2.1 显式配置替代 judge/用户模拟器时,`code_version.json.res |------|--------|---------| | `task_requirements` | verifier.task_requirements | episode 开始时抽取任务需求(1-12 条) | | `plan_step` | planner.plan_step | 每步规划(截图+可见控件+需求+近期动作→下一步决策) | -| `plan_stage` | stage_planner.plan_stage | 仅 stage 模式:初始阶段、阶段完成或请求修复时生成阶段计划 | +| `plan_stage` | stage_planner.plan_stage / goal_planner.plan_goal | stage/goal 的阶段开始或修复;goal 允许读取已存资料后继续规划 | +| `verify_stage` | goal_verifier.verify_goal | goal 阶段结束时独立验证或提取资料增量;明确执行失败的本地检查不算模型调用 | +| `ground_visual` | grounding.ground_visual | goal 按需生成当前截图的有限候选,不直接执行慢模型坐标 | | `verify_observation` | verifier.verify_observation | 每步动作执行后独立验证(BEFORE/AFTER 截图→achieved/not_achieved/uncertain)| `verify_observation` 不信任 planner 的自我宣称:它拿前后截图+预期变化独立裁决,驱动 recent 历史、no_action 连击计数、toggle 检查、以及终态双确认(terminal_claim 模式下逐条核验 requirements + 输出文件存在性)。 @@ -80,6 +88,12 @@ OSWorld 2.1 显式配置替代 judge/用户模拟器时,`code_version.json.res | `stage_local` | 消费已生成的阶段计划;含 stage_goal、milestone 和当帧重绑定后的 plan,不代表新的模型调用 | | `stage_progress` | 里程碑推进,含 stage_goal、milestone、验证 source | | `stage_replan` | 阶段修复请求,含原因及当前里程碑;target_unbound 时另含 `unbound_proposals`、`available_targets`、`recovery_scopes`;不等于调用已经完成 | +| `task_memory` / `memory_update_rejected` | goal 的完整记忆快照/拒绝理由;不是规划模型自行宣称的完成 | +| `goal_action` | 真实回执、当前控件、画面变化与局部效果;effect_verified=false,不等于业务验收通过 | +| `stage_verified` | goal 的阶段裁决、证据引用、结束原因与用时,区分阶段成功和单元完成 | +| `observation_refresh` | 刷新观察而未执行动作,action_replayed=false;不增加动作步数 | +| `recovery` / `goal_stalled` | goal 的视觉候选补全/无可执行进展终止原因 | +| `environment_timing` | setup/observe/execute/export/evaluate 分段耗时 | | `action`(kind 字段) | runner 写的执行记录 `{step, kind: "action", action: {…协议动作…}}` | stage 模式的 planner 事件只在真实规划调用时写入,增加 mode、reason、seconds; diff --git a/docs/STAGE_EXECUTION.md b/docs/STAGE_EXECUTION.md index 3087ce9..5b9135c 100644 --- a/docs/STAGE_EXECUTION.md +++ b/docs/STAGE_EXECUTION.md @@ -17,3 +17,67 @@ PageDown/PageUp 和上/下方向键,覆盖菜单内导航。 `stage_replan` 同时记录 `unbound_proposals`、`available_targets` 和 `recovery_scopes`, 供慢模型在下一次修复中选择不同目标或方法。多个匹配仍然不会被静默猜测。 + +## Goal:带任务记忆的局部连续执行 + +通过 `--execution-mode goal` 选择,默认模式仍为 step,stage 的里程碑流程保持不变。 +该模式从 v7 实验组件迁入,不复制旧环境或覆盖 main 的模型客户端。 + +执行链路:任务要求 → 任务记忆 → 一个有界子目标 → JEV 连续选择当前可执行动作 → +真实回执/新观察 → 阶段结束时独立 Verify → 更新记忆、继续或回到 Planner。 +不会在每个 JEV 动作之后例行调用慢模型 Verify。整题完成仍通过现有独立终态检查。 + +- `planning/goal_planner.py`:抽取交付单元和资料需求,每次规划一个子目标。 +- `memory/state.py`、`memory/context.py`:宿主维护记忆,向 Planner/JEV 提供紧凑视图。 +- `runtime/goal_stage.py`:多步执行、回执核对、局部循环限制和阶段边界。 +- `verification/goal_verifier.py`:阶段验收、源资料增量提取和证据引用校验。 +- `gui/goal_candidates.py`、`gui/grounding.py`:有界动作空间、字段绑定和按需截图候选。 +- `components.py`:通过 memory_factory、stage_verifier 和已有组件字段注入,可分别替换。 + +### 记忆中保存什么 + +| 字段 | 内容与更新规则 | +|---|---| +| requirements | 原始交付要求、约束、单元及 unit_fact_keys;Planner 不能自行降低要求 | +| facts | 已观察的资料、source_ref、evidence_refs、scope、status、complete,可含 entries/observations;未知内容不猜填 | +| materials | 素材来源、已读 covered_parts、摘要和证据,供跨页面继续读取 | +| work_items | 固定单元、depends_on、缺失资料、status、delivery_state、remaining、停滞次数和上次方法 | +| artifacts | 真实工作/目标路径、保存状态与证据,不把文件存在等同于内容正确 | +| failures / execution_constraints | 执行失败、相同状态中的动作循环及无进展导航限制 | +| active_stage / stats | 当前子目标、输入、动作额度和进展计数;另保存更新拒绝记录与证据索引 | + +资料缺一部分不再封锁整个单元:已知部分先制作、保存,需要未知值时再补读。 +阶段增量完成与完整单元完成分开;部分资料不能冒充 complete,部分成果不能冒充 unit_met。 +已确认源事实不能被“当前没看到”覆盖;修正需要明确 correction_reason 和当前已登记证据。 +confirmed 是模型基于证据的判断,不是官方评分保证。内部记忆快照版本不改变消息协议 0.2。 + +### 与 main 的接口 + +- goal 同样经过 semantic_enricher,保留 A11y 的 application/description/scope 等字段及 CDP 融合。 + 没有浏览器快照时继续使用 A11y/截图,不新增浏览器 mutation 或隐藏状态读取。 +- 保留 JEV 的瞬态错误重试、Retry-After、1200 秒恢复窗口和 endpoint_file 地址切换。 + goal 使用联合具体动作选择,operation 排序仅作诊断,不提前剪掉其他动作种类。 +- 文本必须绑定唯一观察字段或截图定位字段;不向任意焦点框暴露所有输入。 +- 连续四次同类导航做一次阶段检查;两次检查均无资料/交付进展时,限制该窗口内此种导航。 + 新资料或交付进展可解除导航限制,不把画面缩放本身当业务完成。 +- Environment.observe 只刷新截图/A11y/CDP,不执行或重放动作,不消耗动作步数,仍消耗墙钟预算。 + 缺少动作回执仍报错,不重放。观察文件有独立帧名,阶段证据在清理临时目录前持久化。 +- API/观察/评分异常继续记 failed_unscored;不把 evaluator、答案或分数送进模型。 + +### 启动与结果 + +唯一私有配置仍是 infra/config.json。harness.execution 可配置 mode、stage_max_actions、 +verify_max_tokens、verify_reasoning_effort、verify_max_chars;goal 的图片请求可配置 +image_quality、max_request_bytes。其余模型、号池、VM 与服务发现沿用原配置。 + +```bash +bash scripts/run_experiment.sh \ + --experiment goal-memory-smoke --tasks 008 --concurrency 1 \ + --slow-model kimi-k3 --slow-key-ids 1,2,3 \ + --execution-mode goal --stage-max-actions 2 --max-steps 6 --max-wall 600 +``` + +这只是链路短测预算,不代表完整解题预算。脚本默认仍为 stage/10 动作,新增参数会写入 manifest。 +事件、回执、CDP 快照和逐角色模型请求沿用原目录;另有 task_memory.json 及 evidence/。 +timing.json 增加环境分段、各角色时间/token 和快模型排序统计;没有 usage 的调用单独计数, +不编造 token。真机检查范围及未通过项见 VALIDATION_CN.md,不把短测零分当完整任务成绩。 diff --git a/docs/VALIDATION_CN.md b/docs/VALIDATION_CN.md new file mode 100644 index 0000000..740ccc9 --- /dev/null +++ b/docs/VALIDATION_CN.md @@ -0,0 +1,90 @@ +# Goal/任务记忆接入验证 + +## 范围 + +2026-10-31,从 AI45Lab/Tandem main 建立独立工作区和 `feature/goal-memory-integration`。 +迁入 v7 的 goal、任务记忆和阶段 Verify,保留 main 的 A11y/CDP 语义融合、协议 0.2、 +API 号池与 JEV 服务恢复。未修改 `integration/components` 分支、既有实验或共享 VM/依赖。 +step/stage 默认路径保留。没有训练模型、没有改评分器、没有加入题号特例。 + +## 本地检查 + +```bash +ruff check src tests demo +PYTHONPATH=src pytest -q +PYTHONPATH=src python -m tandem check +PYTHONPATH=src python -c 'from tandem_eval.config_schema import load_config; load_config("infra/config.json")' +``` + +- Ruff 通过。 +- pytest:404 passed。包含 step/stage/goal mock 全链路、部分资料先交付、记忆保护、字段绑定、 + 重复导航、goal 消费 CDP 元素、只读刷新不重放、联合选择保留 JEV 地址切换与恢复。 +- 协议 0.2:17 个 golden 样例通过;未改 schema/models/validate/samples。 +- 私有配置校验通过;凭据文件保持 600 且被 Git 忽略。 +- 修改的运行时模块均小于 500 行;无新增运行时依赖。 + +上述 synthetic/mock 测试不代表真实任务完成或 benchmark 得分。 + +## 真实 GUI 短测 + +命令从仓库现有脚本启动,仅测试链路,不等待完整任务: + +```bash +bash scripts/run_experiment.sh \ + --experiment goal-memory-gui-smoke-20261031-1806 \ + --tasks 008 --concurrency 1 --slow-model kimi-k3 --slow-key-ids 1,2,3 \ + --execution-mode goal --stage-max-actions 2 --max-steps 6 --max-wall 600 +``` + +这次私有配置为 GUI-only(route.policy=gui、allow_cli=false);CPU worker 使用 +evobox_proxy、private-machine=group、2 CPU、8192 MiB、GPU=0;JEV 复用现有训练服务。 +配置及部署、输出路径与正在运行的 v7 双模型实验独立。脚本可通过 +TANDEM_EXPERIMENT_ROOT 将归档根设置到下列共享目录的 experiments/。 + +共享根目录: +`/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/v0_goal-integration-6step_20261031_smoke` + +- 实际源码:`deploy/`。 +- 轨迹:`rjob_logs/goal-memory-gui-smoke-20261031-1806-008/episodes/008/`。 +- 命令与批次日志:`experiments/goal-memory-gui-smoke-20261031-1806/`。 +- 平台状态、日志及人工停止记录:`manual_platform/`。 + +实际 rjob 为 `tandem-goal-memory-gui-smoke-20261031-1806-0-b3d4b`。 +确认 STOPPED 后已归档并删除该记录,原始轨迹保留;只停止了本次 smoke 的本地 watcher。 + +已验证: + +1. Kimi 需求抽取和 goal 规划均返回 HTTP 200,并产生有效结构化计划。 +2. 取得真实桌面截图、A11y 及 1337 端口 CDP 快照(首次 4 个元素)。9222 的连接失败未阻断采集。 +3. 真实 JEV 连续选择两次向下滚动,排序 used_image=true,耗时约 2.54/2.09 秒。 +4. 两次 GuiAction 均经 VM 实际执行,回执 acknowledged=true、returncode=0; + 两条 traj 的 protocol_version 均为 0.2,并保留 browser_snapshot。 +5. 人工查看执行后截图,真实 LibreOffice Writer 中的报销指南滚动到填写报告说明。 + 不是脚本直接修改文件、mock 或 replay。 +6. 已触发 verify_stage 请求,并保存 task_memory.json。按用户要求在两步后主动停止, + 没有等待 Verify 返回、整题完成或评分,不将停止包装为完整验收通过。 + +## 未通过/未执行的部分 + +- **未执行整题、长任务或准确率回归**。无正式分数,也不能据两次滚动判断性能提升。 +- **真实阶段 Verify 未等待返回**;其契约、记忆提交和阶段出口由本地测试覆盖,尚未完成真机验收。 +- **原生 rjob 名称监控/收尾仍有上游已知问题**:main 使用提交前名称监控,而平台返回带后缀 + 的完整名称。本 PR 不混入通用集群修复;本次按实际名称人工归档和停止。 +- 该短测没有检查真实 GUI 保存/重新打开、跨软件长程恢复或闲时服务抢占。这些不宣称已通过。 +- 按用户最新要求,仅推送 fork 分支,PR 由用户手动创建,不自动合并。 + +## 修改地图 + +- `src/tandem/memory/`:宿主记忆及角色视图。 +- `planning/goal_planner.py`、`verification/goal_verifier.py`、`runtime/goal_stage.py`:规划、执行与验收。 +- `components.py`、`config.py`、`runtime/session.py`:可替换装配和可选 goal 入口。 +- `gui/targets.py`、`goal_candidates.py`、`grounding.py`、`compile.py`:目标恢复与候选编译; + 保留 main 的 UITarget 语义字段及 semantic_enricher。 +- `models/jev.py`:goal 联合具体动作选择,保留 main 的恢复路径; + `models/slow.py`:goal 图片发送预算和阶段 Verify 额度,不替换 main 号池/传输恢复。 +- `tandem_eval/`:仅补 goal 必需的配置传递、只读观察刷新、回执全文和证据/计时落盘; + `browser.py`、评分器、任务清单、共享环境与 `rjob.py` 不改。 +- `scripts/run_experiment.sh`:新增执行模式和阶段额度参数,沿用原默认 stage/10。 +- 配置模板、README、阶段说明、结果目录契约和对应测试同步更新。 + +凭据、截图、轨迹、模型权重和 VM 均不进入 Git;文档只引用共享目录。 diff --git a/infra/config.example.json b/infra/config.example.json index a652093..9dc0486 100644 --- a/infra/config.example.json +++ b/infra/config.example.json @@ -63,7 +63,12 @@ }, "execution": { "mode": "step", - "stage_max_actions": 24 + "stage_max_actions": 24, + "verify_max_tokens": 16384, + "verify_reasoning_effort": null, + "verify_max_chars": 48000, + "image_quality": 85, + "max_request_bytes": 2097152 }, "budget": { "max_steps": 30, diff --git a/scripts/run_experiment.sh b/scripts/run_experiment.sh index 9705f10..5f4729f 100755 --- a/scripts/run_experiment.sh +++ b/scripts/run_experiment.sh @@ -12,6 +12,8 @@ SLOW_MODEL="kimi-k3" SLOW_KEY_IDS="1,2,3,4" MAX_STEPS=500 MAX_WALL=14400 +EXECUTION_MODE=stage +STAGE_MAX_ACTIONS=10 DRY_RUN=0 usage() { @@ -20,6 +22,7 @@ Usage: bash scripts/run_experiment.sh \ --experiment NAME --tasks ID1,ID2,... [--concurrency N] \ [--slow-model NAME] [--slow-key-ids 1,2,3,4] \ + [--execution-mode step|stage|goal] [--stage-max-actions N] \ [--max-steps N] [--max-wall SECONDS] [--dry-run] Experiment names use lowercase letters, digits and hyphens only. @@ -36,6 +39,8 @@ while [[ $# -gt 0 ]]; do --slow-key-ids) SLOW_KEY_IDS="$2"; shift 2 ;; --max-steps) MAX_STEPS="$2"; shift 2 ;; --max-wall) MAX_WALL="$2"; shift 2 ;; + --execution-mode) EXECUTION_MODE="$2"; shift 2 ;; + --stage-max-actions) STAGE_MAX_ACTIONS="$2"; shift 2 ;; --dry-run) DRY_RUN=1; shift ;; -h|--help) usage; exit 0 ;; *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;; @@ -50,6 +55,10 @@ if [[ -z "$TASKS" || "$CONCURRENCY" -lt 1 || "$MAX_STEPS" -lt 1 || "$MAX_WALL" - usage >&2 exit 2 fi +if [[ ! "$EXECUTION_MODE" =~ ^(step|stage|goal)$ || "$STAGE_MAX_ACTIONS" -lt 1 ]]; then + usage >&2 + exit 2 +fi EXPERIMENT_ROOT="${TANDEM_EXPERIMENT_ROOT:-$REPO/rjob_logs}" EXPERIMENT_DIR="$EXPERIMENT_ROOT/$EXPERIMENT" @@ -77,6 +86,7 @@ print(json.loads(Path(sys.argv[1]).read_text())["log"]["results_root"]) PY ) export STAGING EXPERIMENT DRIVER_DIR TASKS CONCURRENCY SLOW_MODEL SLOW_KEY_IDS MAX_STEPS MAX_WALL +export EXECUTION_MODE STAGE_MAX_ACTIONS python3 - <<'PY' import json import os @@ -93,6 +103,8 @@ args = { "slow_key_ids": [int(item) for item in os.environ["SLOW_KEY_IDS"].split(",")], "max_steps": int(os.environ["MAX_STEPS"]), "max_wall_seconds": float(os.environ["MAX_WALL"]), + "execution_mode": os.environ["EXECUTION_MODE"], + "stage_max_actions": int(os.environ["STAGE_MAX_ACTIONS"]), "watch_timeout_seconds": float(os.environ["MAX_WALL"]) + 600, "git_describe": subprocess.check_output( ["git", "describe", "--always", "--dirty"], text=True @@ -124,12 +136,12 @@ run_task() { local driver_log="${DRIVER_DIR}/${task}.driver.log" local rc=0 - tandem-eval rjob run \ + env PYTHONPATH=src python3 -m tandem_eval rjob run \ --run-id "$run_id" \ --benchmark osworld21 \ --tasks "$task" \ - --execution-mode stage \ - --stage-max-actions 10 \ + --execution-mode "$EXECUTION_MODE" \ + --stage-max-actions "$STAGE_MAX_ACTIONS" \ --slow-model "$SLOW_MODEL" \ --slow-key-ids "$SLOW_KEY_IDS" \ --max-steps "$MAX_STEPS" \ diff --git a/src/tandem/components.py b/src/tandem/components.py index 3df6554..54709d0 100644 --- a/src/tandem/components.py +++ b/src/tandem/components.py @@ -1,6 +1,6 @@ """组件契约(依赖注入边界,无插件框架、无隐藏分发)。 -在旧九组件上增加 semantic_enricher,统一 slow planner 与 fast selector 看到的元素语义。 -grounder(visual fallback)默认 None,session 侧判空跳过。compile_gui 产出协议动作。""" +semantic_enricher 统一 planner 与 selector 的元素语义;goal 额外注入任务记忆、阶段验证与 +按需视觉补全。step/stage 装配不变。compile_gui 只产出协议动作,不调用 VM。""" from __future__ import annotations @@ -11,12 +11,16 @@ from .config import SessionConfig from .gui.candidates import ActionCandidateBuilder from .gui.compile import gui_action +from .gui.grounding import ground_visual from .gui.proposals import focused_candidates from .gui.semantics import enrich_targets from .gui.targets import A11yTargetBuilder +from .memory.state import TaskMemory from .models.jev import JevOmniPolicy +from .planning.goal_planner import goal_requirements, plan_goal from .planning.planner import plan_step from .planning.stage_planner import plan_stage +from .verification.goal_verifier import verify_goal from .verification.verifier import task_requirements, verify_observation @@ -32,6 +36,8 @@ class Components: focus_candidates: Callable grounder: Callable | None compile_gui: Callable + memory_factory: Callable | None = None + stage_verifier: Callable | None = None def build_default(*, client, endpoint: str, config: SessionConfig, @@ -42,21 +48,26 @@ def build_default(*, client, endpoint: str, config: SessionConfig, selector = KevPolicy(endpoint, timeout_seconds=config.fast_timeout_seconds) else: selector = JevOmniPolicy(endpoint, timeout_seconds=config.fast_timeout_seconds, - image_stage=image_stage, endpoint_provider=endpoint_provider) + image_stage=image_stage, endpoint_provider=endpoint_provider, + joint_actions=config.execution_mode == "goal") return Components( planner=partial( + plan_goal if config.execution_mode == "goal" else plan_stage if config.execution_mode == "stage" else plan_step, client, allow_cli=config.allow_cli and config.route_policy != "gui", route_policy=config.route_policy, ), verifier=partial(verify_observation, client), - requirements=partial(task_requirements, client), + requirements=partial( + goal_requirements if config.execution_mode == "goal" else task_requirements, client), selector=selector, target_builder=A11yTargetBuilder(), semantic_enricher=enrich_targets, candidate_builder=ActionCandidateBuilder(), focus_candidates=focused_candidates, - grounder=None, + grounder=partial(ground_visual, client) if config.execution_mode == "goal" else None, compile_gui=gui_action, + memory_factory=TaskMemory if config.execution_mode == "goal" else None, + stage_verifier=partial(verify_goal, client) if config.execution_mode == "goal" else None, ) diff --git a/src/tandem/config.py b/src/tandem/config.py index 5786acc..9d4a2c3 100644 --- a/src/tandem/config.py +++ b/src/tandem/config.py @@ -18,15 +18,20 @@ class SessionConfig: execution_mode: str = "step" stage_max_actions: int = 24 slow_max_tokens: int = 16384 + slow_verify_max_tokens: int = 16384 + slow_verify_reasoning_effort: str | None = None + stage_verify_max_chars: int = 48000 slow_timeout_seconds: float = 120 slow_direct_api: bool = False slow_reasoning_effort: str | None = None # None=服务端默认;off=enable_thinking:false + slow_image_quality: int = 85 + slow_max_request_bytes: int = 2 * 1024 * 1024 fast_backend: str = "jev" # "jev"(qwen-jev-choice /rank)| "kev"(Kev System One) fast_timeout_seconds: float = 300 def __post_init__(self): - if self.execution_mode not in {"step", "stage"}: - raise ValueError("execution_mode must be step or stage") + if self.execution_mode not in {"step", "stage", "goal"}: + raise ValueError("execution_mode must be step, stage or goal") if type(self.stage_max_actions) is not int or self.stage_max_actions < 1: raise ValueError("stage_max_actions must be a positive integer") if self.route_policy not in {"auto", "gui", "cli"}: @@ -37,3 +42,5 @@ def __post_init__(self): raise ValueError("fast_backend must be jev or kev") if self.slow_reasoning_effort not in {None, "off", "low", "medium", "high", "max"}: raise ValueError("slow_reasoning_effort must be null/off/low/medium/high/max") + if self.slow_verify_reasoning_effort not in {None, "off", "low", "medium", "high", "max"}: + raise ValueError("slow_verify_reasoning_effort must be null/off/low/medium/high/max") diff --git a/src/tandem/gui/compile.py b/src/tandem/gui/compile.py index c8be177..b03b040 100644 --- a/src/tandem/gui/compile.py +++ b/src/tandem/gui/compile.py @@ -17,10 +17,14 @@ def gui_action( ) -> GuiAction | WaitAction | None: operation = winner.operation metadata = winner.metadata - if operation in {"click", "double_click"} and target is not None: + if operation in {"click", "double_click", "right_click"} and ( + target is not None or metadata.get("visual_point") + ): + point = metadata.get("visual_point") if target is None else [ + target.x + target.width // 2, target.y + target.height // 2] return GuiAction( action_type=operation, - parameters={"x": target.x + target.width // 2, "y": target.y + target.height // 2}, + parameters={"x": point[0], "y": point[1]}, target_id=winner.target_id, description=winner.description, ) @@ -56,7 +60,7 @@ def gui_action( description=winner.description, ) if operation == "type_text": - click = None + click = metadata.get("visual_point") press_f2 = False if target is not None and not metadata.get("focused_input"): click = [target.x + target.width // 2, target.y + target.height // 2] diff --git a/src/tandem/gui/goal_candidates.py b/src/tandem/gui/goal_candidates.py new file mode 100644 index 0000000..6b1a8b7 --- /dev/null +++ b/src/tandem/gui/goal_candidates.py @@ -0,0 +1,187 @@ +"""Current-observation action space for a whole GUI subgoal; no planner click whitelist.""" + +import json +import re +import xml.etree.ElementTree as ET + +from .candidates import ActionCandidate +from .focus import input_and_menu_context +from .targets import _attribute, _local, _pair, _state + + +def matches_field(hint, label): + """Match an intended field to its observed label, not an arbitrary focused control.""" + def words(value): + return set(re.findall(r"\w+", value.casefold())) - {"the", "a", "field", "required"} + + observed = words(label) + return bool(observed) and observed <= words(hint) and not label.startswith(("Unnamed ", "Focused ")) + + +def scroll_point(tree, viewport): + """Anchor wheel events inside the observed foreground window, not the desktop center.""" + windows = [] + for element in ET.fromstring(tree or "").iter(): + if _local(element.tag) not in {"frame", "dialog"} or _state(element, "active") is not True: + continue + position = _pair(_attribute(element, "screencoord")) + size = _pair(_attribute(element, "size")) + if position and size and size[0] > 0 and size[1] > 0: + windows.append((*position, *size)) + x, y, width, height = windows[-1] if windows else (0, 0, *viewport) + return (max(0, min(viewport[0] - 1, x + width * 3 // 5)), + max(0, min(viewport[1] - 1, y + height // 2))) + + +def foreground_tree(raw_tree): + """GNOME's visible app menu disambiguates stale ACTIVE flags on background windows.""" + root = ET.fromstring(raw_tree) + apps = list(root) + shell = next((a for a in apps if a.get("name") == "gnome-shell"), None) + if shell is None: + return raw_tree + overview = [e for e in shell.iter() if e.get("name") == "Overview"] + search_visible = any( + _state(e, "editable") is True and _state(e, "showing") is True + and _pair(_attribute(e, "screencoord")) and _pair(_attribute(e, "size")) + for panel in overview for e in panel.iter()) + if search_visible: + # X11 still names the old window while GNOME's full-screen overview owns input. + for app in apps: + if app is not shell: + root.remove(app) + shell.set("active", "true") + root.set("foreground_title", "GNOME Overview") + root.set("foreground_class", '["gnome-shell"]') + return ET.tostring(root, encoding="unicode") + if root.get("foreground_class"): + def normalize(value): + return re.sub(r"[^a-z0-9]", "", value.casefold()) + + def active(element, value): + key = next((k for k in element.attrib if _local(k) == "active"), "active") + element.set(key, "true" if value else "false") + + classes = [normalize(v) for v in json.loads(root.get("foreground_class")) if v] + title = root.get("foreground_title", "") + for app in apps: + matching = (any(normalize(app.get("name", "")).endswith(c) for c in classes) + or bool(title) and any(e.get("name") == title for e in app.iter() + if _local(e.tag) in {"frame", "dialog"})) + if app is not shell and not matching: + root.remove(app) + elif app is not shell: + windows = [e for e in app.iter() if _local(e.tag) in {"frame", "dialog"}] + named = [e for e in windows if title and e.get("name") == title] + if named: + active(app, False) + for window in windows: + active(window, window in named) + return ET.tostring(root, encoding="unicode") + names = [] + for element in shell.iter(): + attrs = {k.rsplit("}", 1)[-1]: v for k, v in element.attrib.items()} + position = re.findall(r"-?\d+", attrs.get("screencoord", "")) + if (element.tag.rsplit("}", 1)[-1] == "menu" and attrs.get("name") + and attrs.get("showing") == "true" and len(position) == 2 + and 0 <= int(position[1]) < 32): + names.append(attrs["name"].casefold()) + matches = [app for app in apps if app is not shell and any( + name in e.get("name", "").casefold() for e in app.iter() + if e.tag.rsplit("}", 1)[-1] in {"frame", "dialog"} for name in names)] + if len(matches) != 1: + return raw_tree + for app in apps: + if app is not shell and app is not matches[0]: + root.remove(app) + return ET.tostring(root, encoding="unicode") + + +def goal_candidates(space, targets, tree, inputs, viewport=(1920, 1080)): + _, menus = input_and_menu_context(tree or "") + fields = [t for t in targets if t.role in {"text", "textbox", "entry", "searchbox", "spin_button"}] + typed = [] + for index, value in enumerate(inputs): + hint = value.get("target_hint", value["label"]) + matching = [t for t in fields if matches_field(hint, t.name)] + if len(matching) != 1: + continue # Ambiguous/unlabelled fields need an observed click or visual grounding first. + target = matching[0] + if (target.value == value["text"] or value["input_mode"] == "insert" + and value["text"] in target.value): + continue + typed.append(ActionCandidate( + id=f"GOAL_TEXT:{index}", operation="type_text", source="goal_input", target_id=target.id, + description=f"{value['input_mode']} the observed field {target.name!r} with the " + f"supplied {value['label']}: {value['text']!r}", + metadata={"text": value["text"], "input_mode": value["input_mode"], + "target_hint": hint, "field_label": target.name, + "semantic_key": target.semantic_key}, + )) + space.target_heads["type_text"] = typed + unnamed_entries = [t for t in targets if t.name.startswith(("Unnamed ", "Focused ")) + and t.role in {"text", "textbox", "entry", "searchbox"}][:2] + present = {c.target_id for c in space.target_heads["click"]} + space.target_heads["click"] = [ActionCandidate( + id="FOCUS:" + t.id, operation="click", target_id=t.id, source="a11y", + description=f"Focus the observed {t.name!r} in {' / '.join(t.parent_context)} before typing.", + metadata={"semantic_key": t.semantic_key}) for t in unnamed_entries if t.id not in present + ] + space.target_heads["click"] + native_files = [t for t in targets if t.role in {"canvas", "label"} + and "nautilus" in (t.application + " ".join(t.parent_context)).lower()][:4] + space.target_heads["double_click"].extend(ActionCandidate( + id="OPEN:" + t.id, operation="double_click", target_id=t.id, source=t.source, + description=f"Open the observed file-manager item {t.name!r}.", + metadata={"semantic_key": t.semantic_key}) for t in native_files) + sliders = [t for t in targets if t.role == "slider"][:2] + space.target_heads["set_slider"] = [ActionCandidate( + id=f"SLIDER:{t.id}:{fraction}", operation="set_slider", target_id=t.id, + description=f"Set {t.name!r} to {fraction:.0%} of its range" + + (f" (value {t.range_min + fraction * (t.range_max - t.range_min):.2f})" + if t.range_min is not None and t.range_max is not None else "") + + f"; current observed value: {t.value or 'unknown'}.", + metadata={"fraction": fraction, "semantic_key": t.semantic_key}) + for t in sliders for fraction in (0.1, 0.25, 0.5, 0.75, 0.9) + if (not t.value or t.range_min is None or t.range_max is None + or abs(float(t.value) - (t.range_min + fraction * (t.range_max - t.range_min))) + > max(1e-6, (t.range_max - t.range_min) * 0.02))] + # A slider's default center click is ambiguous and was repeatedly seeking to the same midpoint. + slider_ids = {t.id for t in sliders} + space.target_heads["click"] = [c for c in space.target_heads["click"] if c.target_id not in slider_ids] + space.target_heads["right_click"] = [ActionCandidate( + id="RIGHT:" + c.target_id, operation="right_click", source=c.source, target_id=c.target_id, + description="Open the context menu for " + c.description, + metadata=dict(c.metadata)) for c in space.target_heads["click"][:8]] + sx, sy = scroll_point(tree, viewport) + for candidate in space.target_heads["scroll"]: + candidate.metadata.update(scroll_x=sx, scroll_y=sy) + for menu in menus[:16]: + space.target_heads["keypress"].append(ActionCandidate( + id="MENU_KEY:" + menu["path"], operation="keypress", source="a11y_menu", + description="Open the observed menu path " + menu["path"], + metadata={"key_sequence": menu["key_sequence"]})) + space.target_heads["keypress"].extend(ActionCandidate( + id="GOAL_KEY:" + "+".join(keys), operation="keypress", source="global", + description=description, metadata={"key_sequence": [keys]}) for keys, description in [ + (["ctrl", "s"], "Save the current document."), + (["ctrl", "o"], "Open the application's file chooser for an existing document."), + (["alt", "tab"], "Switch away from the foreground window to another open application."), + (["shift", "f10"], "Open the focused object's context menu."), + (["ctrl", "l"], "Focus the browser or native file-dialog location field."), + (["ctrl", "f"], "Open the current application's Find field."), + (["shift", "tab"], "Move focus to the previous field."), + (["down"], "Move down in the current selection or menu."), + (["up"], "Move up in the current selection or menu."), + (["right"], "Move right in the current selection, calendar or menu."), + (["left"], "Move left in the current selection, calendar or menu."), + ]) + space.controls = {k: v for k, v in space.controls.items() + if k in {"wait", "done", "visual_fallback", "return_to_planner"}} + space.controls["done"] = ActionCandidate( + id="STAGE_DONE", operation="done", source="stage_control", + description="The current subtask's completion criteria are satisfied. Request ONE independent " + "stage verification; this does not finish the original task.") + space.controls["return_to_planner"].description = ( + "The current subtask needs unavailable information, new reasoning or an unsupported operation. " + "Return control with existing progress preserved; do not repeat an ineffective action.") + return space diff --git a/src/tandem/gui/grounding.py b/src/tandem/gui/grounding.py new file mode 100644 index 0000000..b7a65a2 --- /dev/null +++ b/src/tandem/gui/grounding.py @@ -0,0 +1,66 @@ +"""Existing screenshot-grounded target fallback, reused without changing the JEV service.""" + +import base64 +import json + +from PIL import Image + +from .candidates import ActionCandidate +from .goal_candidates import matches_field + + +def ground_visual(client, instruction, plan, screenshot): + with Image.open(screenshot) as image: + width, height = image.size + proposals = plan["action_proposals"] + value = client.json_request([ + {"role": "system", "content": ( + "Ground the immediate GUI intent in the CURRENT screenshot. Propose up to four distinct " + "visible target alternatives, never a screen grid or a sequence. Do not execute anything. " + "Return JSON {candidates:[{operation,target,bounds,text,input_mode,target_hint}]}. " + "Operations: click, double_click, right_click, type_text, scroll_up, scroll_down. " + "bounds=[x,y,width,height] in " + "ORIGINAL screenshot pixels. Ground only foreground unobscured targets. For type_text copy exact " + "text/input_mode AND target_hint from ONE supplied input, and ground the visible field named " + "by that hint. target must be its observed label/identity, not the text to enter. Never put " + "unrelated text in the currently focused field; otherwise click to focus first. Use [] if no " + "required target is visible. Do not invent targets or completion evidence." + )}, + {"role": "user", "content": [ + {"type": "text", "text": json.dumps({"task": instruction, "local_plan": plan, + "screenshot_size": [width, height]}, ensure_ascii=False)}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + + base64.b64encode(screenshot.read_bytes()).decode("ascii")}}, + ]}, + ], lambda v: isinstance(v.get("candidates"), list) + and all(isinstance(i, dict) for i in v["candidates"]), role="ground_visual") + allowed_text = {(p.get("text"), p.get("input_mode"), p.get("target_hint", p.get("label", ""))) + for p in proposals if p.get("operation") == "type_text"} + candidates = [] + for item in value["candidates"][:4]: + operation, box = item.get("operation"), item.get("bounds") + if operation not in {"click", "double_click", "right_click", "type_text", "scroll_up", "scroll_down"}: + continue + if not isinstance(box, list) or len(box) != 4 or any(type(v) is not int for v in box): + continue + x, y, w, h = box + if not (0 <= x < x + w <= width and 0 <= y < y + h <= height): + continue + point = [x + w // 2, y + h // 2] + metadata = {"visual_point": point, "visual_bounds": box} + if operation == "type_text": + if ((item.get("text"), item.get("input_mode"), item.get("target_hint")) not in allowed_text + or not matches_field(item.get("target_hint", ""), item.get("target", ""))): + continue + metadata.update(text=item["text"], input_mode=item["input_mode"], + target_hint=item["target_hint"], field_label=item["target"]) + name = f"VISUAL:{operation}:{x}:{y}:{w}:{h}" + if operation.startswith("scroll_"): + name = f"{operation.upper()}:{name}" + operation = "scroll" + metadata.update(scroll_x=point[0], scroll_y=point[1]) + candidates.append(ActionCandidate( + id=name, operation=operation, source="visual_grounder", metadata=metadata, + description=f"{item['operation']} screenshot-visible {item.get('target', 'control')!r} " + f"at box {box}." + (f" Text: {metadata['text']!r}." if "text" in metadata else ""))) + return candidates diff --git a/src/tandem/gui/targets.py b/src/tandem/gui/targets.py index 0f7bd8a..9b9f915 100644 --- a/src/tandem/gui/targets.py +++ b/src/tandem/gui/targets.py @@ -33,6 +33,8 @@ class UITarget: scope: str = "native" label_source: str = "a11y" operations: tuple[str, ...] = () + range_min: float | None = None + range_max: float | None = None def compact(self) -> dict: return {key: value for key, value in asdict(self).items() if value not in (None, "", (), False)} @@ -100,6 +102,32 @@ def _attribute(element: ET.Element, name: str) -> str: return "" +def _number(element, name): + value = _attribute(element, name) + return float(value) if re.fullmatch(r"-?\d+(?:\.\d+)?", value) else None + + +def _is_overlay(element): + return _local(element.tag) in {"dialog", "alert", "popup_menu"} or ( + _local(element.tag) == "menu" and _attribute(element, "ispopup") in {"true", "auto", "menu"}) + + +def _field_labels(root): + labels = {} + for parent in root.iter(): + preceding = "" + for child in parent: + if _local(child.tag) in {"label", "static"}: + preceding = str(child.get("name") or child.text or "").strip() + continue + fields = [e for e in child.iter() if _state(e, "editable") is True + or _local(e.tag) in {"entry", "textbox", "searchbox", "combo_box", "spin_button"}] + if preceding and len(fields) == 1: + labels[fields[0]] = preceding + preceding = "" + return labels + + class A11yTargetBuilder: def build( self, @@ -108,22 +136,40 @@ def build( max_targets: int = 64, infer_button_labels: bool = False, include_sliders: bool = False, + context: str = "", + filter_occluded: bool = False, ) -> list[UITarget]: root = ET.fromstring(raw_tree) targets: list[UITarget] = [] + field_labels = _field_labels(root) if filter_occluded else {} + target_nodes, overlays = {}, {} + if filter_occluded: + for node in root.iter(): + if (_is_overlay(node) and _state(node, "showing") is True + and _state(node, "visible") is not False): + position, size = _pair(_attribute(node, "screencoord")), _pair(_attribute(node, "size")) + if position and size and min(size) > 0: + overlays.setdefault((*position, *size), set()).update(node.iter()) - def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name: str) -> None: + def visit(element, active, parents, app_name, modal=False, roles=(), file_manager=False): active = active or _state(element, "active") is True role = _local(element.tag) + modal = modal or _is_overlay(element) raw_name = str(element.attrib.get("name") or "").strip() name = raw_name if role == "application": app_name = name label_source = "a11y" if name else "unresolved" + if not name and element in field_labels: + name, label_source = field_labels[element], "a11y_adjacent" + if (not name and filter_occluded and "gnome-shell" in parents + and "Overview" in parents and _state(element, "editable") is True): + name, label_source = "Desktop search", "a11y_context" + file_manager = file_manager or "nautilus" in app_name.lower() or "file chooser" in name.lower() value = ( str(element.text or "").strip() if role in {"text", "textbox", "entry", "searchbox", "spin_button"} - else "" + else _attribute(element, "value") if role == "slider" else "" ) position = _pair(_attribute(element, "screencoord")) size = _pair(_attribute(element, "size")) @@ -149,6 +195,9 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name: and active and ( any("file chooser" in item.lower() or item.lower() == "places" for item in parents) + or filter_occluded and file_manager and any(r in { + "list_item", "tree_item", "button", "push_button", "menu_item", "option", "link"} + for r in roles) or bool( re.search( r"\b(?:monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b", name, re.I @@ -158,7 +207,8 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name: ) if ( (active or system_target) - and (role in SUPPORTED_ROLES or safe_label or (include_sliders and role == "slider")) + and (role in SUPPORTED_ROLES or safe_label or (include_sliders and role == "slider") + or filter_occluded and (role == "image" or file_manager and role == "canvas")) and (name or role in SUPPORTED_ROLES) and interactive_text and enabled is not False @@ -168,7 +218,6 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name: and size and size[0] > 0 and size[1] > 0 - and len(targets) < max_targets ): normalized_name = " ".join(name.lower().split()) parent_context = tuple(item for item in parents[-3:] if item) @@ -198,10 +247,24 @@ def visit(element: ET.Element, active: bool, parents: tuple[str, ...], app_name: or _attribute(element, "placeholder-text") ), label_source=label_source, + source="a11y_dialog" if filter_occluded and modal else "a11y", + range_min=_number(element, "min") if role == "slider" else None, + range_max=_number(element, "max") if role == "slider" else None, ) ) + target_nodes[targets[-1].id] = element for child in element: - visit(child, active, context[-4:], app_name) + visit(child, active, context[-4:], app_name, modal, (*roles, role), file_manager) visit(root, False, (), "") - return targets + if filter_occluded: + targets = [t for t in targets if not any( + target_nodes[t.id] not in nodes and x <= t.x + t.width // 2 < x + width + and y <= t.y + t.height // 2 < y + height + for (x, y, width, height), nodes in overlays.items())] + if context: + words = set(re.findall(r"\w+", context.lower())) + targets.sort(key=lambda t: (t.source == "a11y_dialog", + len(words & set(re.findall(r"\w+", t.name.lower()))), t.focused is True, + t.role in {"button", "push_button", "combo_box", "entry", "textbox"}), reverse=True) + return targets[:max_targets] diff --git a/src/tandem/memory/__init__.py b/src/tandem/memory/__init__.py new file mode 100644 index 0000000..e6c80da --- /dev/null +++ b/src/tandem/memory/__init__.py @@ -0,0 +1 @@ +"""Task-scoped source facts, delivery progress and execution evidence.""" diff --git a/src/tandem/memory/context.py b/src/tandem/memory/context.py new file mode 100644 index 0000000..d930597 --- /dev/null +++ b/src/tandem/memory/context.py @@ -0,0 +1,59 @@ +"""Compact role-specific views. Raw receipts/screenshots remain in the episode record.""" + + +def _fact_view(fact, limit=2400): + view = {k: v for k, v in fact.items() if k not in {"observations", "entries"}} + if len(view["value"]) > limit: + view["value"] = (view["value"][:limit // 2] + + "\n[excerpt; request read_memory for the stored content]\n" + + view["value"][-limit // 2:]) + view["value_excerpt_only"] = True + return view + + +def planning_context(memory): + ready = memory.ready_items()[:4] + keys = {f["key"] for item in ready for f in ( + memory.preparation_target(item)[0]["facts_needed"] if item["kind"] == "prepare" + else item["facts_needed"])} + facts = [f for f in memory.facts.values() if f["scope"] == "task"] + auxiliary = {f["key"] for f in facts[-8:]} + visible = [_fact_view(f) for f in facts if f["key"] in keys | auxiliary] + return {"requirements": [{k: r[k] for k in ("id", "requirement", "kind", "status")} + for r in memory.requirements], + "ready_work_items": [{**item, "can_start_partial": item["kind"] == "deliver", + "next_acquisition": { + "unit": memory.preparation_target(item)[0]["goal"], + "missing_facts": memory.preparation_target(item)[1]}} + for item in ready], + "completed_work": [i["id"] for i in memory.work_items.values() if i["status"] == "done"], + "known_facts": visible, + "fact_index": [{**{k: f[k] for k in ("key", "status", "source_ref")}, + "complete": f.get("complete", True)} for f in facts], + "historical_ui_notes": [f for f in memory.facts.values() if f["scope"] == "stage"][-6:], + "materials": [{"source_ref": m["source_ref"], "covered_parts": m["covered_parts"], + "summary_excerpt": m["summary"][:1800], + "summary_excerpt_only": len(m["summary"]) > 1800} + for m in list(memory.materials.values())[-8:]], + "artifacts": list(memory.artifacts.values()), "failures": memory.failures[-12:], + "execution_constraints": memory.execution_constraints[-12:], + "memory_update_rejections": memory.last_rejections, + "stages_without_progress": memory.stats["stages_without_progress"]} + + +def execution_context(memory): + stage = memory.active_stage + item = memory.work_items[stage["work_item_id"]] + return {"goal": stage["goal"], "completion_criteria": stage["completion_criteria"], + "method": stage["method"], + "constraints": [r["requirement"] for r in memory.requirements if r.get("kind") == "constraint"], + "inputs": stage["inputs"], "guidance": stage["guidance"], + "target_application": stage.get("target_application"), + "facts": [_fact_view(memory.facts[k]) for k in stage["input_fact_keys"] if k in memory.facts], + "delivery_state": item["delivery_state"], "remaining": item["remaining"], + "failures": [f for f in memory.failures if f["work_item_id"] == item["id"]][-4:], + "execution_constraints": [c for c in memory.execution_constraints + if c["work_item_id"] == item["id"]][-6:], + "actions_remaining": stage["max_actions"] - stage["actions"], + "control": "Choose each next action from the NEW observation. DONE requests stage verification, " + "not whole-task completion. Do not repeat satisfied input or toggle states."} diff --git a/src/tandem/memory/state.py b/src/tandem/memory/state.py new file mode 100644 index 0000000..a5734a8 --- /dev/null +++ b/src/tandem/memory/state.py @@ -0,0 +1,229 @@ +"""Host-owned task memory. Plans request work; stage verification commits progress.""" + +import json +from copy import deepcopy + +LEVELS = {"none": 0, "implemented": 1, "saved": 2, "verified": 3} + + +class TaskMemory: + def __init__(self, instruction, requirements): + self.instruction = instruction + self.requirements = deepcopy(requirements) + self.facts = {} + self.materials = {} + self.work_items = {} + self.artifacts = {} + self.failures = [] + self.execution_constraints = [] + self.active_stage = None + self.stage_count = 0 + self.revision = 0 + self.known_evidence = set() + self.last_rejections = [] + sources = {f["key"]: f for r in self.requirements for f in r["facts_needed"]} + self.source_keys = set(sources) + self.stats = {"stages": 0, "verified_stages": 0, "gui_actions": 0, + "cli_actions": 0, "new_facts": 0, "delivery_advances": 0, + "source_parts": 0, "stages_without_progress": 0} + for req in self.requirements: + req.update(status="unknown", evidence_refs=[]) + if req.get("kind") == "constraint": + continue + needed = [sources[key] for key in dict.fromkeys( + key for keys in req["unit_fact_keys"] for key in keys)] + preparation = f"{req['id']}:prepare" + if needed: + self._add(preparation, req, "prepare", + "Acquire missing source facts for a ready unit", [], needed) + for index, unit in enumerate(req["units"], 1): + keys = req["unit_fact_keys"][index - 1] + self._add(f"{req['id']}:unit:{index}", req, "deliver", unit, [], + [sources[key] for key in keys]) + + def _add(self, key, req, kind, goal, dependencies, facts): + self.work_items[key] = { + "id": key, "requirement_ids": [req["id"]], "kind": kind, "goal": goal, + "completion_criteria": goal, "depends_on": dependencies, + "facts_needed": facts, "status": "pending", + "delivery_state": "none", "remaining": [], "evidence_refs": [], + } + + def _facts_ready(self, item): + return all(self.facts.get(f["key"], {}).get("status") == "confirmed" + and self.facts[f["key"]]["scope"] == "task" + and self.facts[f["key"]].get("complete", True) for f in item["facts_needed"]) + + def preparation_target(self, item): + units = [item] if item["kind"] == "deliver" else [ + w for w in self.work_items.values() if w["kind"] == "deliver" + and w["status"] != "done" and w["requirement_ids"] == item["requirement_ids"]] + choices = [(unit, [f for f in unit["facts_needed"] + if not self._facts_ready({"facts_needed": [f]})]) for unit in units] + choices = [(unit, missing) for unit, missing in choices if missing] + return min(choices, key=lambda pair: len(pair[1])) if choices else (item, []) + + def ready_items(self): + for item in self.work_items.values(): + if item["kind"] == "prepare" and item["status"] != "done" and self._facts_ready(item): + item.update(status="done", evidence_refs=[ref for f in item["facts_needed"] + for ref in self.facts[f["key"]]["evidence_refs"]]) + return sorted([i for i in self.work_items.values() if i["status"] != "done" + and all(self.work_items[d]["status"] == "done" for d in i["depends_on"])], + key=lambda i: (i["kind"] != "deliver", i.get("stalled_stages", 0), + len(self.preparation_target(i)[1]))) + + def start(self, plan, step): + item = self.work_items[plan["work_item_id"]] + if item not in self.ready_items(): + raise ValueError("Stage work item is not ready or has already completed") + self.stage_count += 1 + self.active_stage = {**deepcopy(plan), "id": f"S{self.stage_count}", + "started_step": step, "actions": 0, "status": "running"} + if plan["purpose"] == "prepare": + unit, missing = self.preparation_target(item) + self.active_stage.update(prepares_unit=unit["id"], fact_targets=deepcopy(missing)) + item["status"] = "active" + self.stats["stages"] += 1 + return self.active_stage + + def record_action(self, record): + self.active_stage["actions"] += 1 + self.stats["cli_actions" if record["kind"] == "cli" else "gui_actions"] += 1 + if record["returncode"] != 0: + error = (record.get("stderr") or record.get("stdout") or "Execution failed")[-700:] + self._failure(error, [record["evidence_ref"]]) + + def _failure(self, error, refs): + stage = self.active_stage + matches = [f for f in self.failures if f["work_item_id"] == stage["work_item_id"] + and f["error"] == error] + if matches: + matches[-1]["count"] += 1 + matches[-1]["evidence_refs"] = refs + else: + self.failures.append({"stage_id": stage["id"], "work_item_id": stage["work_item_id"], + "method": stage["method"], "error": error, + "count": 1, "evidence_refs": refs}) + + def apply(self, result, evidence): + """Only the fixed active unit is writable; references must belong to real evidence.""" + stage = self.active_stage + item = self.work_items[stage["work_item_id"]] + current = set(evidence) + self.known_evidence.update(current) + allowed = self.known_evidence + self.last_rejections = [] + advanced = False + for fact in result["facts"]: + fact = deepcopy(fact) + refs = fact["evidence_refs"] + if not refs or not set(refs) <= allowed: + self.last_rejections.append({"key": fact["key"], "reason": "unregistered_evidence"}) + continue + if fact["key"] in self.source_keys: + fact["scope"] = "task" + old = self.facts.get(fact["key"]) + entries = fact.get("entries", {}) + old_entries = old.get("entries", {}) if old else {} + conflicts = any(key in old_entries and old_entries[key] != value + for key, value in entries.items()) + extending = bool(old_entries and entries and not conflicts) + correction = bool(fact.get("correction_reason") and set(refs) & current) + # Incomplete prose sources may arrive a section at a time without structured entries. + # Append evidence, never replace the previously observed section or unlock a partial fact. + if (old and old["status"] == fact["status"] == "confirmed" + and old["scope"] == fact["scope"] == "task" and not old.get("complete", True) + and not entries and not old_entries and set(refs) & current and not correction): + sections = deepcopy(old.get("observations", [{k: old[k] for k in + ("value", "source_ref", "evidence_refs")}])) + if fact["value"] not in [section["value"] for section in sections]: + sections.append({k: fact[k] for k in ("value", "source_ref", "evidence_refs")}) + fact["observations"] = sections + fact["value"] = "\n".join(section["value"] for section in sections) + fact["evidence_refs"] = list(dict.fromkeys(old["evidence_refs"] + refs)) + extending = True + if entries: + fact["entries"] = {**old_entries, **entries} + fact["value"] = json.dumps(fact["entries"], ensure_ascii=False) + fact["evidence_refs"] = list(dict.fromkeys((old or {}).get("evidence_refs", []) + refs)) + fact["complete"] = bool(fact.get("complete", False) or (old or {}).get("complete", False)) + if (old and old["status"] == "confirmed" and old["scope"] == "task" + and (fact["status"] != "confirmed" or fact["scope"] != "task" + or old.get("complete", True) and not fact.get("complete", True) + or old["value"] != fact["value"] and not (correction or extending) + or conflicts and not correction)): + self.last_rejections.append({"key": fact["key"], "reason": "confirmed_source_preserved"}) + continue + if (fact["status"] == "confirmed" and fact["scope"] == "task" + and (old is None or old["value"] != fact["value"] or old["status"] != fact["status"] + or old.get("complete", True) != fact.get("complete", True))): + advanced = advanced or fact["key"] in self.source_keys + self.stats["new_facts"] += 1 + self.facts[fact["key"]] = {**fact, "requirement_ids": item["requirement_ids"], + "observed_stage": stage["id"]} + for material in result["materials"]: + if material["evidence_refs"] and set(material["evidence_refs"]) <= allowed: + old = self.materials.get(material["source_ref"], {}) + new_parts = set(material["covered_parts"]) - set(old.get("covered_parts", [])) + if new_parts: + self.stats["source_parts"] += len(new_parts) + summary = old.get("summary", "") + if material["summary"] not in summary: + summary = (summary + "\n" + material["summary"]).strip() + self.materials[material["source_ref"]] = { + **material, "summary": summary, + "covered_parts": list(dict.fromkeys(old.get("covered_parts", []) + + material["covered_parts"])), + "evidence_refs": list(dict.fromkeys(old.get("evidence_refs", []) + + material["evidence_refs"]))} + refs = result["evidence_refs"] + grounded = bool(set(refs) & current) and set(refs) <= allowed + level = result["delivery_state"] + delivering = item["kind"] == "deliver" and stage["purpose"] != "prepare" + if (grounded and delivering and LEVELS[level] > LEVELS[item["delivery_state"]] + and (level != "verified" or result["stage_outcome"] == "met")): + item["delivery_state"] = level + advanced = True + self.stats["delivery_advances"] += 1 + item.update(status="pending", remaining=result["remaining"], evidence_refs=refs if grounded else []) + preparation_met = item["kind"] == "prepare" and self._facts_ready(item) + if grounded and result["stage_outcome"] == "met": + self.stats["verified_stages"] += 1 + if preparation_met or (delivering and result["unit_met"] + and level == "verified"): + item["status"] = "done" + advanced = True + for check in result["requirement_checks"]: + if check["evidence_refs"] and set(check["evidence_refs"]) <= allowed: + req = next((r for r in self.requirements if r["id"] == check["id"]), None) + if req is not None: + req.update(status=check["status"], evidence_refs=check["evidence_refs"]) + for artifact in result["artifacts"]: + if not artifact["evidence_refs"] or not set(artifact["evidence_refs"]) <= allowed: + continue + old = self.artifacts.get(artifact["target_path"], {}) + self.artifacts[artifact["target_path"]] = {**artifact, "revision": old.get("revision", 0) + 1} + for key in result["invalidated_work_item_ids"]: + if key in self.work_items and self.work_items[key]["kind"] == "deliver" and grounded: + self.work_items[key].update(status="pending", delivery_state="none", + remaining=["Contradictory evidence requires reinspection"]) + if result["stage_outcome"] != "met" and grounded: + self._failure("; ".join(result["remaining"]) or result["summary"], refs) + self.stats["stages_without_progress"] = 0 if advanced else self.stats["stages_without_progress"] + 1 + item["stalled_stages"] = 0 if advanced else item.get("stalled_stages", 0) + 1 + item["last_attempt"] = {k: stage[k] for k in ("route", "purpose", "method", "command")} + self.revision += 1 + self.active_stage = None + self.ready_items() + return advanced + + def snapshot(self): + return deepcopy({"schema_version": 2, "revision": self.revision, "instruction": self.instruction, + "requirements": self.requirements, "materials": list(self.materials.values()), + "facts": list(self.facts.values()), "work_items": list(self.work_items.values()), + "artifacts": list(self.artifacts.values()), "failures": self.failures, + "execution_constraints": self.execution_constraints, + "memory_update_rejections": self.last_rejections, + "active_stage": self.active_stage, "stats": self.stats, + "evidence_refs": sorted(self.known_evidence)}) diff --git a/src/tandem/models/jev.py b/src/tandem/models/jev.py index 18950c9..342b2b8 100644 --- a/src/tandem/models/jev.py +++ b/src/tandem/models/jev.py @@ -21,11 +21,12 @@ class JevOmniPolicy: def __init__(self, endpoint: str, timeout_seconds: float = 300, opener=None, - image_stage: tuple[str, str] | None = None, endpoint_provider=None): + image_stage: tuple[str, str] | None = None, endpoint_provider=None, joint_actions=False): self.endpoint = str(endpoint).rstrip("/") self.timeout_seconds = timeout_seconds self.opener = opener or urllib.request.build_opener(urllib.request.ProxyHandler({})) self.endpoint_provider = endpoint_provider + self.joint_actions = joint_actions # (写视角目录, 排序服务宿主机读视角目录):/rank 要求 image_path 在服务端可读, # 截图本体在 adpc 时先暂存到 evobox 中转目录(按内容哈希命名去重)。 self.image_stage = image_stage @@ -164,6 +165,18 @@ def predict(choice_request: dict[str, Any]) -> tuple[list[float], bool]: candidate for candidate in candidates if candidate.operation == selected_operation ] action_request = action_requests[selected_operation] + if self.joint_actions: + # Keep the operation ranking diagnostic; do not prune other concrete goal actions. + descriptions = {choice_id: option for request in action_requests.values() + for choice_id, option in zip( + request["option_ids"], request["options"], strict=True)} + selected_candidates = candidates + action_request = { + "state": operation_request["state"], + "question": "Which complete concrete action should be performed next across ALL operations?", + "option_ids": [c.id for c in candidates], + "options": [f"{c.operation}: {descriptions[c.id]}" for c in candidates], + } if len(selected_candidates) == 1: action_scores, action_used_image = [1.0], operation_used_image else: diff --git a/src/tandem/models/slow.py b/src/tandem/models/slow.py index 35dcaf9..aab11a3 100644 --- a/src/tandem/models/slow.py +++ b/src/tandem/models/slow.py @@ -5,18 +5,78 @@ from __future__ import annotations +import base64 +import io import json import time import urllib.error import urllib.request from pathlib import Path +from PIL import Image + from .key_pool import ApiKeyPool, PoolExhausted, classify_api_error, retry_after_seconds # 空正文(全留在思考通道)或截断时,下一次重试升档思考强度,避免原样重发重复失败 _REASON_EFFORT_LADDER = {"low": "medium", "medium": "high"} +def _pack_images(payload, quality, max_bytes): + """Pack full-resolution send copies; only exactly identical decoded frames share an image. + + Mutate the existing messages list so validators can still append correction turns on retry. + Never write source files or discard distinct evidence frames to meet the body budget. + """ + frames, pixels = [], [] + stats = {"image_count": 0, "duplicate_images": 0, "source_image_bytes": 0} + for message in payload["messages"]: + if not isinstance(message.get("content"), list): + continue + for part in message["content"]: + if part.get("type") != "image_url": + continue + url = part["image_url"]["url"] + if not url.startswith("data:image/"): + continue + raw = base64.b64decode(url.split(",", 1)[1]) + with Image.open(io.BytesIO(raw)) as source: + frame = source.convert("RGB") + identity = (frame.size, frame.tobytes()) + stats["image_count"] += 1 + stats["source_image_bytes"] += len(raw) + if identity in pixels: + ordinal = pixels.index(identity) + 1 + part.clear() + part.update(type="text", text=f"Same screenshot as supplied image #{ordinal} (exact pixels).") + stats["duplicate_images"] += 1 + else: + pixels.append(identity) + frames.append((part, frame)) + while frames: + image_bytes = 0 + for part, frame in frames: + encoded = io.BytesIO() + frame.save(encoded, format="JPEG", quality=quality, subsampling=0, optimize=True) + raw = encoded.getvalue() + image_bytes += len(raw) + part["image_url"]["url"] = "data:image/jpeg;base64," + base64.b64encode(raw).decode("ascii") + stats.update(image_quality=quality, sent_image_bytes=image_bytes) + if len(json.dumps(payload).encode("utf-8")) <= max_bytes or quality == 65: + break + quality = max(65, quality - 10) + while frames and len(json.dumps(payload).encode("utf-8")) > max_bytes and max( + max(frame.size) for _, frame in frames + ) > 960: + for part, frame in frames: + frame.thumbnail((max(960, int(frame.width * 0.8)), max(540, int(frame.height * 0.8)))) + encoded = io.BytesIO() + frame.save(encoded, format="JPEG", quality=quality, subsampling=0, optimize=True) + part["image_url"]["url"] = "data:image/jpeg;base64," + base64.b64encode( + encoded.getvalue()).decode("ascii") + return {**stats, "unique_images": len(frames), "sent_image_sizes": [list(f.size) for _, f in frames]} + + + def extract_json_object(text: str): """从模型正文中抢救 JSON:剥围栏后,若剩余文本不是纯 JSON,抽取首个平衡的 {…}。 返回解析后的 dict;无 JSON 或解析失败返回 None。""" @@ -84,13 +144,19 @@ def json_request(self, messages, validate, *, role): "model": credentials.get("model") or "unknown", "messages": messages, "stream": False, - "max_tokens": self.config.slow_max_tokens, + "max_tokens": (min(self.config.slow_max_tokens, self.config.slow_verify_max_tokens) + if role == "verify_stage" else self.config.slow_max_tokens), } effort = self.config.slow_reasoning_effort + if role == "verify_stage" and self.config.slow_verify_reasoning_effort is not None: + effort = self.config.slow_verify_reasoning_effort if effort == "off": payload["enable_thinking"] = False elif effort is not None: payload["reasoning_effort"] = effort + image_stats = (_pack_images(payload, self.config.slow_image_quality, + self.config.slow_max_request_bytes) + if self.config.execution_mode == "goal" else {}) def build_request(): return urllib.request.Request( @@ -125,6 +191,7 @@ def audit(event, started, attempt, **details): "time": time.time(), "elapsed_seconds": round(time.perf_counter() - started, 3), "request_bytes": len(request.data), + **image_stats, **details, } text = json.dumps(self.key_pool.redact(record), ensure_ascii=False) diff --git a/src/tandem/planning/goal_planner.py b/src/tandem/planning/goal_planner.py new file mode 100644 index 0000000..805145e --- /dev/null +++ b/src/tandem/planning/goal_planner.py @@ -0,0 +1,258 @@ +"""One self-contained subgoal, not a speculative list of future UI actions.""" + +import base64 +import json + +from ..memory.context import _fact_view, planning_context + +REQUIREMENTS = """Extract the ORIGINAL task into JSON {requirements:[{id,requirement,output_path, +kind,units,unit_fact_keys,facts_needed}]}. kind is delivery or constraint. units is a list of STRINGS with +delivery units with their completion criteria; enumerate explicitly requested pages/items, otherwise +use one unit. Constraints have units:[]. facts_needed is [{key,description}] naming only essential +stable source information acquired from supplied materials; reuse the SAME key across requirements. +Never register current focus, open windows or selection as required source facts. Split large source +packets into the concrete fields or small sections needed by individual deliverables. unit_fact_keys +is an array parallel to units: each entry lists ONLY the facts_needed keys required by THAT unit. +Keys may reference definitions in ANY requirement; define each shared source key once. Use [] for a +unit needing no source reading. Constraints MUST have units:[] AND unit_fact_keys:[], not [[]]. +Include a constraint's required source key on the delivery units that must satisfy that constraint. +A source-reading dependency must not gate unrelated work. +Do not list facts already stated in the instruction as facts_needed. Names, titles, URLs and target +systems explicitly provided by the user are inputs, not missing source facts. +Keep all original quantities, locations, prohibitions, interaction requirements and saving conditions. +Enumerate the individual pages/slides/form sections as small delivery units, including when their count +is explicit but their names are not. Do not collapse 'all 26 slides' into one giant unit. Each unit +lists only its own missing source content; whole-document consistency remains a final constraint. +output_path is null unless literally given as an absolute output path in the instruction. +Do not prescribe actions, guess source facts, inspect evaluators, or introduce hidden grading criteria. +Return a compact contract, no reasoning, no speculative extra work.""" + +REQUIREMENTS += ''' +Every requirement MUST contain ALL seven keys, including a plain-string "requirement" describing the +actual obligation (also for constraints). units MUST be an array of STRINGS, never objects. +Exact shape example: {"requirements":[{"id":"R1","requirement":"Create and save the requested document", +"output_path":null,"kind":"delivery","units":["Requested document is complete and saved"], +"unit_fact_keys":[[]], +"facts_needed":[]}]}. This example illustrates the format only; extract the real task's obligations. +''' + +PLAN = """Delegate ONE meaningful bounded subtask using the host's task memory. You are the planner; +the fast GUI choice model will autonomously observe and act, not follow an action list. Return JSON: +{done:false,work_item_id:"exact ready id",route:"gui|cli",purpose:"prepare|produce|save|verify", +goal:"specific result",completion_criteria:"stage exit condition",method:"concise approach", +input_fact_keys:["known key"],inputs:[{label:"field meaning",target_hint:"observed field label", +text:"exact supplied text", +input_mode:"replace|insert"}],guidance:"brief necessary navigation/context, not an ordered click script", +max_actions:24,command:"CLI only; otherwise empty string"}. +Use a ready work item's FIXED goal and original requirements. Do not rename units or lower final criteria. +Missing source facts are acquisition targets, NOT a hard gate on starting a delivery unit. First make +and save the useful portion supported by known facts; leave unknown values unresolved, never guess them. +Choose a bounded incremental stage exit condition, not the whole unit's acceptance criteria. A partial +stage may succeed while the unit remains incomplete. Read a missing source only before the next action +actually needs it, then resume the preserved artifact. Respect explicit user-required reading order. +input_fact_keys may name ONLY confirmed task-scope keys present in memory.known_facts, NOT keys that +are merely requested in facts_needed. Use [] before any source facts have been observed. +For prepare, acquire only missing essential facts; do not reread facts already in memory. Source notes +are historical evidence, not guaranteed current UI. Reuse confirmed task facts. Temporary inventory, +selection and page state must be reobserved. Do not fabricate a URL or missing personal information. +Bound visual source reading to one small section at a time (normally 4-6 observations); the stage-end +verifier will record its contents. Do not delegate reading several documents and navigating a new site +as a single preparation goal. Later preparation continues from materials.covered_parts, not page one. +For GUI give a whole local goal (a form section, adding an attachment, a settings operation), normally +8-24 possible actions, not merely 'open menu'. It may span pages/dialogs; new targets must be observed, +not guessed in advance. Provide ALL exact text values it will need from the instruction or known facts. +For every text input, target_hint must identify its intended observed field (copy the visible label +when available). For browser navigation use the observed address-bar label; for a document body or +desktop launcher identify that specific editor/search field. Text is never interchangeable among fields. +The choice model cannot generate free text. It can click, type provided inputs, use keyboard, scroll, +wait, request stage verification, or return control. Missing data/new reasoning requires another stage. +For CLI provide one bounded Bash command, current inputs and expected real results. Prefer actually +producing/saving deliverables over repeatedly reading tutorials, building audits or protecting unchanged +files. Inspect edited documents in the actual application before claiming them verified. Keep source +and final paths distinct. CLI is forbidden if allow_cli=false. Task constraints override routing choices. +Do not inspect evaluator code, ground truth, rewards, hidden state, unrelated secrets or scoring APIs. +Previous methods/errors are in memory; changing their description does not make them new. If stalled, +choose another method or another independent ready unit. Avoid optional research and cosmetic detours. +Only return {done:true,task_status:"completed|infeasible",current_goal:"evidence-based reason"} for an +actual whole-task terminal claim. Stage completion is NOT task completion. A blocked unit is not proof +the task is infeasible. Keep the response short: goal, inputs and one bounded command, no long analysis.""" + +PLAN += """ +Prefer a ready delivery unit over more preparation. Preparation is bounded acquisition for ONE missing +field or source section, not an all-document audit or a pixel-analysis research project. State the exact +observable subgoal, never 'Acquire the required source facts'. Reuse saved extractions and artifacts. +After two stages without new required facts or delivery progress, use a genuinely different method or +another ready unit. Do not add optional diagnostic outputs as new completion requirements. Make and +save a usable increment as soon as its inputs are known; preserve all original final requirements. +historical_ui_notes are prior observations, NOT the current interface. Current screenshot/controls win. +For prepare, follow next_acquisition: acquire the stated missing registered fields for that ONE unit. +Do not inventory every dropdown or inspect source code before filling fields whose values are known. +Use exact registered keys for acquired contents; fact_index lists already observed auxiliary sources, +not new dependencies. Do not reread auxiliary sources solely because their full text is omitted here. +Never type into an unrelated/Untitled document. Confirm the target application and document first. +For editing an existing document, include target_document:"exact filename" on GUI produce/save stages. +If it is not currently open in the foreground, first use a prepare stage to open/switch to it. A +navigation-only prepare stage may have no source fact targets. For websites or creating a NEW document, +target_document may be null. Available GUI operations include right-click, double-click, Escape, +Alt+Tab, Ctrl+O, keyboard navigation, supplied text, scrolling and screenshot-grounded targets. +known_facts includes partial and auxiliary source content; complete:false is usable for a partial +increment but does NOT mean the source or unit is complete. Never guess ownership/identity when joining +records from different sources. Do not keep preparing solely to make every facts_needed entry complete. +If an observed fact is indexed but its text is omitted/excerpted, return ONLY {read_memory:["key",...]} +for up to six keys, at most twice before planning. This reads stored evidence without reopening files. +Failure constraints, remaining work and rejected memory updates are actionable state, not suggestions +to repeat the same attempt with different wording. When the executor reports a blocked cycle, choose +another interaction, another source or an independent ready unit. +For a GUI stage that needs to open or switch applications, include target_application:"exact app name". +This explicit input will also be available for typing in the desktop launcher; do not leave it only in +prose guidance. Set it to null when not applicable. Prefer CLI to open an installed application or a +known document when allowed, then delegate actual UI editing to JEV. Do not alternate the application +drawer open/closed or repeat a covered target. Resolve the visible overlay or use another interaction. +""" + + +def goal_requirements(client, instruction): + messages = [{"role": "system", "content": REQUIREMENTS}, {"role": "user", "content": instruction}] + + def valid(v): + okay = (isinstance(v.get("requirements"), list) and bool(v["requirements"]) + and all(isinstance(r, dict) and isinstance(r.get("id"), str) + and isinstance(r.get("requirement"), str) + and r.get("kind") in {"delivery", "constraint"} + and isinstance(r.get("units"), list) + and (r.get("kind") == "constraint" or bool(r["units"])) + and all(isinstance(u, str) for u in r["units"]) + and isinstance(r.get("unit_fact_keys"), list) + and len(r["unit_fact_keys"]) == len(r["units"]) + and all(isinstance(keys, list) and all(isinstance(k, str) for k in keys) + for keys in r["unit_fact_keys"]) + and isinstance(r.get("facts_needed"), list) + and all(isinstance(f, dict) and isinstance(f.get("key"), str) + and isinstance(f.get("description"), str) for f in r["facts_needed"]) + for r in v["requirements"])) + if okay: + registered = {f["key"] for r in v["requirements"] for f in r["facts_needed"]} + okay = all(set(keys) <= registered for r in v["requirements"] for keys in r["unit_fact_keys"]) + if not okay: + messages.append({"role": "user", "content": + "Invalid requirements JSON. Every entry needs id, requirement (plain text, mandatory), " + "output_path, kind, units (array of STRINGS, not objects), unit_fact_keys " + "(one array of source key strings per unit) and facts_needed " + "(array of {key,description}). Include constraint text in requirement; units:[] is not " + "a substitute. Constraints need units:[] AND unit_fact_keys:[], not [[]]. " + "Shared source keys may be defined in any requirement. Return the COMPLETE object " + "{requirements:[...]}, not an isolated fact or requirement."}) + return okay + + value = client.json_request(messages, valid, role="task_requirements") + for r in value["requirements"]: + path = r.get("output_path") + if not isinstance(path, str) or not path.startswith("/") or path not in instruction: + r["output_path"] = None + return value["requirements"] + + +def goal_error(value, memory, allow_cli): + """Name the rejected field/constraint so JSON repair can actually change the request.""" + if type(value.get("done")) is not bool: + return "done must be a boolean" + if value["done"]: + return ("" if value.get("task_status") in ("completed", "infeasible") + and isinstance(value.get("current_goal"), str) and value["current_goal"].strip() + else "terminal claims require task_status and a nonempty current_goal") + ready = {i["id"]: i for i in memory.ready_items()} + if not isinstance(value.get("work_item_id"), str): + return "work_item_id must be a string naming an exact ready unit" + item = ready.get(value.get("work_item_id")) + if item is None: + return "work_item_id must be one of " + json.dumps(list(ready)) + if item.get("stalled_stages", 0) >= 2: + previous = item.get("last_attempt", {}) + if (value.get("route") == previous.get("route") == "cli" + and isinstance(value.get("command"), str) + and value["command"].strip() == previous.get("command", "").strip()): + return "command repeats the same CLI attempt after two stalled stages; change method or work unit" + if value.get("route") not in (("gui", "cli") if allow_cli else ("gui",)): + return "route must be gui" + (" or cli" if allow_cli else "; CLI is disabled") + purposes = ("prepare",) if item["kind"] == "prepare" else ("prepare", "produce", "save", "verify") + if value.get("purpose") not in purposes: + return "purpose must be one of " + json.dumps(sorted(purposes)) + for key in ("goal", "completion_criteria", "method"): + if not isinstance(value.get(key), str) or not value[key].strip(): + return key + " must be a nonempty string" + if not isinstance(value.get("guidance"), str): + return "guidance must be a string (empty allowed)" + for key in ("target_document", "target_application"): + if value.get(key) is not None and (not isinstance(value[key], str) or not value[key].strip()): + return key + " must be a nonempty string or null" + if type(value.get("max_actions")) is not int or not 1 <= value["max_actions"] <= 24: + return "max_actions must be an integer in 1..24" + allowed = [k for k, f in memory.facts.items() if f["status"] == "confirmed" and f["scope"] == "task"] + if not isinstance(value.get("input_fact_keys"), list) or any( + not isinstance(k, str) or k not in allowed for k in value["input_fact_keys"]): + return "input_fact_keys must contain only acquired confirmed task keys: " + json.dumps(allowed) + if not isinstance(value.get("inputs"), list): + return "inputs must be an array (empty allowed)" + for index, entry in enumerate(value["inputs"]): + if (not isinstance(entry, dict) or not isinstance(entry.get("label"), str) + or not isinstance(entry.get("text"), str) + or entry.get("input_mode") not in ("replace", "insert")): + return f"inputs[{index}] requires label:string, text:string, input_mode:replace|insert" + if "target_hint" in entry and (not isinstance(entry["target_hint"], str) + or not entry["target_hint"].strip()): + return f"inputs[{index}].target_hint must identify the intended field" + if not isinstance(value.get("command"), str): + return "command must be a string; use an empty string for GUI" + if value["route"] == "cli" and not value["command"].strip(): + return "command must be a nonempty Bash command for CLI" + return "" + + +def valid_goal(value, memory, allow_cli): + return not goal_error(value, memory, allow_cli) + + +def plan_goal(client, instruction, screenshot, targets, recent, tree, verified_focus=None, + requirements=None, remaining_steps=None, *, memory, allow_cli=True, route_policy="auto"): + context = {"task": instruction, "memory": planning_context(memory), "allow_cli": allow_cli, + "route_policy": route_policy, "remaining_actions": remaining_steps, + "visible_controls": [t.compact() for t in targets], "recent": recent[-4:]} + messages = [{"role": "system", "content": PLAN}, {"role": "user", "content": [ + {"type": "text", "text": json.dumps(context, ensure_ascii=False)}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + + base64.b64encode(screenshot.read_bytes()).decode("ascii")}}, + ]}] + reads = 0 + + def validate(v): + if set(v) == {"read_memory"}: + okay = (reads < 2 and isinstance(v["read_memory"], list) and 1 <= len(v["read_memory"]) <= 6 + and all(isinstance(k, str) and k in memory.facts and memory.facts[k]["scope"] == "task" + for k in v["read_memory"])) + error = ("read_memory needs 1..6 existing task-scope keys and at most two reads" + if not okay else "") + else: + error = goal_error(v, memory, allow_cli) + if error: + messages.append({"role": "user", "content": + "Invalid goal-stage JSON: " + error + ". Return the COMPLETE corrected compact stage " + "object, preserving evidence and original requirements. Do not replay an executed action."}) + return not error + + while True: + value = client.json_request(messages, validate, role="plan_stage") + if set(value) != {"read_memory"}: + application = value.get("target_application") + if not value["done"] and value["route"] == "gui" and application and not any( + i["text"] == application for i in value["inputs"]): + value["inputs"].append({"label": "Target application name for desktop search", + "target_hint": "Desktop search", "text": application, + "input_mode": "replace"}) + return value + reads += 1 + messages.append({"role": "assistant", "content": json.dumps(value)}) + messages.append({"role": "user", "content": json.dumps({ + "stored_source_facts": [_fact_view(memory.facts[k], 16000) for k in value["read_memory"]], + "instruction": "Use the stored evidence; now return a bounded stage plan. " + "Do not reopen these sources just to recover the same content.", + "memory_reads_remaining": 2 - reads}, ensure_ascii=False)}) diff --git a/src/tandem/runtime/goal_stage.py b/src/tandem/runtime/goal_stage.py new file mode 100644 index 0000000..17247a0 --- /dev/null +++ b/src/tandem/runtime/goal_stage.py @@ -0,0 +1,401 @@ +"""Goal-stage loop: receipts/observations each step, one semantic verifier at stage end.""" + +import json +import re +import time +import xml.etree.ElementTree as ET +from pathlib import Path +from urllib.parse import unquote, urlsplit + +from PIL import Image, ImageChops, ImageStat + +from ..gui.goal_candidates import foreground_tree, goal_candidates +from ..memory.context import execution_context +from ..models.ranking import ranking_dict +from ..protocol.models import CliAction, FailAction +from .episode import handle_done, resolve_done_probe + + +class ObservationRequired(Exception): + """Refresh without executing/replaying an action or consuming an action budget.""" + + +def _state_key(targets): + return tuple(sorted((t.role, t.name, t.x, t.y, t.value, bool(t.focused), str(t.checked), str(t.selected)) + for t in targets)) + + +def _visual_changed(before, after): + frames = [] + for path in (before, after): + with Image.open(path) as image: + frames.append(image.crop((0, min(32, image.height // 4), image.width, image.height)) + .convert("L").resize((160, 90))) + return ImageStat.Stat(ImageChops.difference(*frames)).mean[0] > 1.0 + + +def _loop_state(targets, tree): + root = ET.fromstring(tree or "") + return (root.get("foreground_title", ""), root.get("foreground_class", ""), + tuple(sorted((t.role, _control_name(t), t.x, t.y, "" if t.role == "slider" else t.value, + str(t.checked)) for t in targets))) + + +def _control_name(target): + if target.role != "page_tab": + return target.name + name = re.sub(r"\s+- Memory usage - .+$", "", target.name) + return re.sub(r"\s*[–-]\s*(?:Audio playing|Video playing in picture-in-picture mode)$", "", name) + + +def _cycle_actions(recent, state): + for period in (2, 3, 4): + if len(recent) < period * 2: + continue + first, second = recent[-period * 2:-period], recent[-period:] + if first == second and any(a != first[0] for a in first): + return {a["key"] for a in second if a["state"] == state} + return set() + + +def _action_key(candidate, targets=()): + target = next((t for t in targets if t.id == candidate.target_id), None) + name = (f"{target.role}|{_control_name(target)}|{target.x}|{target.y}" if target else + candidate.metadata.get("semantic_key") or candidate.description) + if candidate.operation == "keypress": + name += repr([(t.role, t.name, t.x, t.y) for t in targets if t.focused]) + parameters = {k: candidate.metadata[k] + for k in ("text", "input_mode", "fraction", "key_sequence", "visual_point") + if k in candidate.metadata} + return (candidate.operation, name, json.dumps(parameters, sort_keys=True, ensure_ascii=False)) + + +def _target_effect(old, targets): + """Local effect evidence, not semantic stage acceptance. Unknown never means success.""" + if old["operation"] == "type_text": + focus = [t for t in targets if t.focused and t.role != "password_text"] + if len(focus) == 1 and focus[0].value: + if old.get("field_label") and focus[0].name.casefold() != old["field_label"].casefold(): + return "not_met" + matches = (focus[0].value == old["expected_text"] if old["input_mode"] == "replace" + else old["expected_text"] in focus[0].value) + return "met" if matches else "not_met" + if old["operation"] == "set_slider": + target = next((t for t in targets if t.semantic_key == old["target_semantic"]), None) + if (target and target.value and target.range_min is not None and target.range_max is not None + and target.range_max > target.range_min): + fraction = (float(target.value) - target.range_min) / (target.range_max - target.range_min) + return "met" if abs(fraction - old["fraction"]) <= 0.04 else "not_met" + return "unknown" + + +def _document_mismatch(document, tree, targets): + """Reject observed contradictions, not a website's human title or a generic app title.""" + title = ET.fromstring(tree or "").get("foreground_title", "") + filename = Path(document).name.casefold() + if Path(document).stem.casefold() in title.casefold(): + return False + addresses = [t.value for t in targets if "address" in t.name.casefold() + and "://" in t.value] + if Path(document).suffix.casefold() in {".html", ".htm"} and addresses: + return all(Path(unquote(urlsplit(value).path)).name.casefold() != filename for value in addresses) + return bool(re.search(r"\buntitled\b|\S+\.(?:pptx?|docx?|xlsx?|od[ptsg]|pdf)\b", title, re.I)) + + +def _navigation_key(candidate, target, work, state): + if candidate.operation == "scroll": + return (work, *state[:2], "scroll", "up" if "UP" in candidate.id.upper() else "down") + if target and re.search(r"\b(?:zoom|pan)\b", target.name, re.I): + return (work, *state[:2], candidate.operation, target.name.casefold()) + return None + + +class GoalExecution: + def __init__(self, memory, max_actions): + self.memory = memory + self.max_actions = max_actions + self.pending = None + self.history = [] + self.end_reason = None + self.visual = [] + self.visual_step = None + self.attempts = [] + self.failed_actions = {} + self.navigation_failures = {} + self.grounded_states = set() + self.no_effect = 0 + self.empty_stages = 0 + + def _emit_state(self, session, step): + session.events.append({"route": "task_memory", "step": step, + "state": self.memory.snapshot()}) + + def _receive(self, session, observation, targets, tree): + if self.pending is None: + return + receipt = observation.last_receipt + if receipt is None or not receipt.acknowledged: + raise RuntimeError("Missing execution receipt; action is not replayed") + old = self.pending + self.pending = None + signature = _state_key(targets) + changed = old["kind"] == "cli" or signature != old["state"] or _visual_changed( + old["before_screenshot"], observation.screenshot.path) + effect = _target_effect(old, targets) + relevant_change = (old["kind"] == "cli" or effect == "met" + or effect != "not_met" and old["loop_state"] != _loop_state(targets, tree)) + if effect == "unknown" and (not targets or old.get("visual_action")): + relevant_change = changed + if old["kind"] != "cli": + key = (self.memory.active_stage["work_item_id"], old["loop_state"], old["candidate_key"]) + if not relevant_change: + self.failed_actions[key] = self.failed_actions.get(key, 0) + 1 + self.attempts.append({"work": key[0], "state": old["loop_state"], "key": old["candidate_key"]}) + self.attempts = self.attempts[-16:] + self.no_effect = 0 if relevant_change else self.no_effect + 1 + record = {"step": observation.step_idx, "evidence_ref": f"A{observation.step_idx}", + "kind": old["kind"], "operation": old["operation"], "target": old["target"], + "action": old["action"], "returncode": receipt.returncode, + "stdout": receipt.stdout, "stderr": receipt.stderr, + "full_output_ref": receipt.full_output_ref, "screenshot": observation.screenshot.path, + "observed_controls": [t.compact() for t in targets][:40], + "source_frame": self.memory.active_stage["purpose"] == "prepare", + "signature": signature, "observed_change": changed, + "relevant_change": relevant_change, "target_effect": effect, + "navigation_key": old.get("navigation_key")} + self.history.append(record) + self.memory.record_action(record) + session.recent.append({"operation": old["operation"], "target": old["target"], + "goal": self.memory.active_stage["goal"], + "outcome": "changed" if relevant_change else "no_effect", + "returncode": receipt.returncode, + "evidence_ref": record["evidence_ref"]}) + session.events.append({"route": "goal_action", "step": observation.step_idx, + "record": record, "effect_verified": False}) + if receipt.returncode != 0: + self.end_reason = "execution_error" + elif old["kind"] == "cli": + self.end_reason = "cli_returned" + elif self.memory.active_stage["actions"] >= self.memory.active_stage["max_actions"]: + self.end_reason = "stage_action_limit" + elif (record["navigation_key"] and len(self.history) >= 4 + and all(h.get("navigation_key") == record["navigation_key"] for h in self.history[-4:])): + self.end_reason = "repeated_navigation_checkpoint" + elif self.no_effect >= 3 and (session.components.grounder is None + or (self.memory.active_stage["work_item_id"], _loop_state(targets, tree)) + in self.grounded_states): + self.end_reason = "repeated_action_without_observable_progress" + + def _finish(self, session, observation): + reference = f"{self.memory.active_stage['id']}:O{observation.step_idx}" + evidence = {h["evidence_ref"]: h for h in self.history} + evidence[reference] = {"screenshot": observation.screenshot.path} + started = time.perf_counter() + result = session.components.stage_verifier( + session.instruction, Path(observation.screenshot.path), memory=self.memory, + history=[{k: v for k, v in h.items() if k != "signature"} for h in self.history], + end_reason=self.end_reason, evidence=evidence) + stage_id = self.memory.active_stage["id"] + work = self.memory.active_stage["work_item_id"] + source_progress = (self.memory.stats["new_facts"], self.memory.stats["source_parts"]) + advanced = self.memory.apply(result, evidence) + acquired = source_progress != (self.memory.stats["new_facts"], self.memory.stats["source_parts"]) + if advanced or acquired: + self.navigation_failures = {k: v for k, v in self.navigation_failures.items() if k[0] != work} + self.memory.execution_constraints = [c for c in self.memory.execution_constraints + if not (c["work_item_id"] == work + and "navigation_key" in c)] + elif self.end_reason == "repeated_navigation_checkpoint": + key = self.history[-1]["navigation_key"] + self.navigation_failures[key] = self.navigation_failures.get(key, 0) + 1 + self.empty_stages = self.empty_stages + 1 if not self.history and not advanced and not acquired else 0 + if self.memory.work_items[work]["status"] == "done": + self.failed_actions = {k: v for k, v in self.failed_actions.items() if k[0] != work} + self.attempts = [a for a in self.attempts if a["work"] != work] + self.memory.execution_constraints = [c for c in self.memory.execution_constraints + if c["work_item_id"] != work] + for rejected in self.memory.last_rejections: + session.events.append({"route": "memory_update_rejected", "stage_id": stage_id, **rejected}) + session.events.append({"route": "stage_verified", "step": observation.step_idx, + "stage_id": stage_id, "end_reason": self.end_reason, + "evidence": {k: {"screenshot": v["screenshot"], + "full_output_ref": v.get("full_output_ref")} + for k, v in evidence.items()}, + "verification": result, "memory_advanced": advanced, + "seconds": time.perf_counter() - started}) + session.recent.append({"operation": "stage_verified", "stage_id": stage_id, + "outcome": result["stage_outcome"], "evidence": result["summary"], + "remaining": result["remaining"]}) + self.history, self.end_reason, self.visual = [], None, [] + self.visual_step = None + self.no_effect = 0 + self._emit_state(session, observation.step_idx) + if self.empty_stages >= 3: + session.events.append({"route": "goal_stalled", "step": observation.step_idx, + "reason": "three_consecutive_stages_without_an_action"}) + return FailAction(reason="three_consecutive_stages_without_an_action") + raise ObservationRequired("stage_verification_returned") + + def act(self, session, observation): + tree = session._read_tree(observation) + if tree: + tree = foreground_tree(tree) + screenshot = Path(observation.screenshot.path) + targets = session.components.target_builder.build( + tree, f"t{observation.step_idx}", 128, infer_button_labels=True, include_sliders=True, + context=" ".join(str(v) for k, v in (self.memory.active_stage or {}).items() + if k in {"goal", "completion_criteria", "method", "guidance"} + ) or session.instruction, filter_occluded=True, + ) if tree else [] + targets = session.components.semantic_enricher( + targets, tree, (observation.screenshot.width, observation.screenshot.height), + session._read_browser_snapshot(observation)) + self._receive(session, observation, targets, tree) + if self.end_reason: + return self._finish(session, observation) + if session.done_probe is not None: + result = resolve_done_probe(session, observation, tree) + if result is not None: + return result + raise ObservationRequired("terminal_rejected") + if self.memory.active_stage is None: + started = time.perf_counter() + plan = session.components.planner( + session.instruction, screenshot, targets, session.recent, tree or "", + requirements=session.requirements, remaining_steps=observation.budget.steps_remaining, + memory=self.memory) + session.events.append({"route": "planner", "step": observation.step_idx, + "mode": "goal", "plan": plan, + "seconds": time.perf_counter() - started}) + if plan["done"]: + result = handle_done(session, plan, observation, tree) + if result is not None: + return result + raise ObservationRequired("terminal_rejected") + plan["max_actions"] = min(plan["max_actions"], self.max_actions) + self.memory.start(plan, observation.step_idx) + self._emit_state(session, observation.step_idx) + raise ObservationRequired("planner_returned") + stage = self.memory.active_stage + document = stage.get("target_document") + if (stage["route"] == "gui" and stage["purpose"] in {"produce", "save"} + and stage["actions"] == 0 and document and _document_mismatch(document, tree, targets)): + self.end_reason = f"target_document_mismatch: expected {document!r} in the observed foreground" + return self._finish(session, observation) + if stage["route"] == "cli": + action = CliAction(command=stage["command"], timeout_s=session.config.cli_timeout_seconds, + want_observation="screenshot") + self.pending = {"kind": "cli", "operation": "cli", "target": stage["method"], + "action": action.to_dict(), "state": _state_key(targets), + "before_screenshot": observation.screenshot.path} + session.events.append({"route": "cli", "step": observation.step_idx, + "stage_id": stage["id"], "action": action.to_dict()}) + return action + return self._gui(session, observation, targets, tree, screenshot) + + def _gui(self, session, observation, targets, tree, screenshot): + context = execution_context(self.memory) + stage = self.memory.active_stage + state = _loop_state(targets, tree) + work = stage["work_item_id"] + blocked_navigation = {k for k, count in self.navigation_failures.items() + if k[:3] == (work, *state[:2]) and count >= 2} + for key in blocked_navigation: + constraint = {"work_item_id": work, "window": list(state[:2]), "navigation_key": list(key), + "reason": "Repeated navigation changed pixels but two checkpoints acquired no " + "source content or delivery progress. Use another interaction/source."} + if constraint not in self.memory.execution_constraints: + self.memory.execution_constraints.append(constraint) + context["avoid_navigation"] = [list(k) for k in blocked_navigation] + recent = [a for a in self.attempts if a["work"] == work][-8:] + cycle_keys = _cycle_actions(recent, state) + cycle = bool(cycle_keys) + blocked = {key[2] for key, count in self.failed_actions.items() + if key[:2] == (work, state) and count >= 2} + if cycle: + blocked.update(cycle_keys) + for key in blocked: + failure_key = (work, state, key) + self.failed_actions[failure_key] = max(2, self.failed_actions.get(failure_key, 0)) + for key in blocked: + constraint = {"work_item_id": work, "window": list(state[:2]), "action_key": list(key), + "reason": "Repeated action/cycle in the same observed controls. Try a different " + "interaction; unrelated animation or new facts do not clear it."} + if constraint not in self.memory.execution_constraints: + self.memory.execution_constraints.append(constraint) + context["avoid_actions"] = [list(key) for key in blocked] + semantic_missing = not any("gnome-shell" not in t.parent_context for t in targets) + need_grounding = ((semantic_missing or self.no_effect >= 2 or cycle) + and (work, state) not in self.grounded_states) + if need_grounding and session.components.grounder is not None: + self.grounded_states.add((work, state)) + self._ground(session, observation, screenshot, context) + if self.visual: + raise ObservationRequired("local_visual_recovery_returned") + # No visual target does not remove valid keyboard recovery such as Escape or Alt+Tab. + space = session.components.candidate_builder.build( + targets, 96, action_inputs=[], context=" ".join( + context[k] for k in ("goal", "completion_criteria", "method", "guidance"))) + space = goal_candidates(space, targets, tree, context["inputs"], + (observation.screenshot.width, observation.screenshot.height)) + if self.visual_step == observation.step_idx and self.visual: + space.target_heads["visual"] = self.visual + if (work, state) in self.grounded_states: + space.controls.pop("visual_fallback", None) + for operation in space.target_heads: + space.target_heads[operation] = [c for c in space.target_heads[operation] + if _action_key(c, targets) not in blocked and _navigation_key( + c, next((t for t in targets if t.id == c.target_id), None), work, state) + not in blocked_navigation] + space.controls = {k: c for k, c in space.controls.items() if _action_key(c, targets) not in blocked} + ranking = session.components.selector.rank( + stage_goal=context["goal"], intent=json.dumps(context, ensure_ascii=False), + active_window=ET.fromstring(tree).get("foreground_title", "") if tree else "", + targets=targets, action_space=space, + recent_actions=session.recent[-8:], screenshot=screenshot) + winner = ranking.winner + operation = winner.operation if winner else "return_to_planner" + target = next((t for t in targets if winner and t.id == winner.target_id), None) + action = session.components.compile_gui(winner, target) if winner else None + session.events.append({"route": "gui", "step": observation.step_idx, "mode": "goal", + "stage_id": self.memory.active_stage["id"], + "plan": {"stage_goal": context["goal"], "current_goal": context["goal"]}, + "candidates": [c.compact() for c in space.candidates()], + "winner": winner.compact() if winner else None, + "avoid_actions": context["avoid_actions"], + "ranking": ranking_dict(ranking), + "action": action.to_dict() if action else None}) + if operation == "visual_fallback" and session.components.grounder is not None: + if (work, state) not in self.grounded_states: + self.grounded_states.add((work, state)) + self._ground(session, observation, screenshot, context) + if self.visual: + raise ObservationRequired("visual_grounder_returned") + if action is None: + self.end_reason = "stage_done_requested" if operation == "done" else "returned_to_planner" + return self._finish(session, observation) + self.pending = {"kind": action.kind, "operation": operation, + "target": winner.description, "action": action.to_dict(), + "state": _state_key(targets), "loop_state": state, + "candidate_key": _action_key(winner, targets), + "before_screenshot": observation.screenshot.path, + "expected_text": winner.metadata.get("text"), + "input_mode": winner.metadata.get("input_mode"), + "target_semantic": target.semantic_key if target else None, + "field_label": winner.metadata.get("field_label"), + "navigation_key": _navigation_key(winner, target, work, state), + "visual_action": operation == "scroll" or winner.source == "visual_grounder" + or target is not None and target.role == "canvas", + "fraction": winner.metadata.get("fraction")} + self.visual, self.visual_step = [], None + return action + + def _ground(self, session, observation, screenshot, context): + plan = {**context, "current_goal": context["goal"], "stage_goal": context["goal"], + "action_proposals": [{"operation": "type_text", **v} for v in context["inputs"]]} + self.visual = session.components.grounder(session.instruction, plan, screenshot) + self.visual_step = observation.step_idx + session.events.append({"route": "recovery", "step": observation.step_idx, + "stage_id": self.memory.active_stage["id"], + "method": "visual_grounding", "candidate_count": len(self.visual), + "avoid_actions": context.get("avoid_actions", [])}) diff --git a/src/tandem/runtime/session.py b/src/tandem/runtime/session.py index 460337c..31fe6e4 100644 --- a/src/tandem/runtime/session.py +++ b/src/tandem/runtime/session.py @@ -18,6 +18,7 @@ from ..models.ranking import ranking_dict from ..protocol.models import Action, CliAction, FailAction, GuiAction, Observation, WaitAction from .episode import handle_done, resolve_done_probe, verify_pending +from .goal_stage import GoalExecution from .stages import StageExecution, recovery_scopes from .stopping import NO_ACTION_LIMIT, budget_exhausted @@ -30,6 +31,7 @@ def __init__(self, components: Components, *, config: SessionConfig | None = Non self.begin(requirements=[]) # 构造即重置,但不触发 requirements 抽取 def begin(self, instruction: str = "", requirements: list | None = None) -> None: + self.goal = None self.instruction = instruction # requirements=None(真实 run)时走组件抽取(旧 runtime 同款); # 显式 [](mock 链路)跳过 LLM 调用 @@ -45,9 +47,14 @@ def begin(self, instruction: str = "", requirements: list | None = None) -> None self.stage = ( StageExecution(self.config.stage_max_actions) if self.config.execution_mode == "stage" else None ) + if self.config.execution_mode == "goal": + self.goal = GoalExecution(self.components.memory_factory(instruction, requirements), + self.config.stage_max_actions) # ------------------------------------------------------------------ 主入口 def act(self, observation: Observation) -> Action: + if self.goal is not None: + return self.goal.act(self, observation) tree = self._read_tree(observation) if self.pending is not None: verify_pending(self, observation, tree) diff --git a/src/tandem/verification/goal_verifier.py b/src/tandem/verification/goal_verifier.py new file mode 100644 index 0000000..1306c7e --- /dev/null +++ b/src/tandem/verification/goal_verifier.py @@ -0,0 +1,247 @@ +"""One requirement-bound semantic verification per ended goal stage, never per GUI action.""" + +import base64 +import json +from pathlib import Path + +PROMPT = """Independently verify the ENDED desktop subtask against its fixed work unit and ORIGINAL +requirements. Do not judge each intermediate action, demand later pages earlier, or trust the actor's +printed success. Return only JSON: +{stage_outcome:"met|not_met|uncertain",unit_met:false,delivery_state:"none|implemented|saved|verified", +summary:"specific observed result",remaining:["concrete missing part"],evidence_refs:["supplied id"], +facts:[{key:"registered or concrete reusable key",value:"exact content including numbers/units", +status:"confirmed|uncertain",scope:"task|stage",source_ref:"actual file/page/location", +evidence_refs:["supplied id"],complete:true}], +materials:[{source_ref:"actual file/url",covered_parts:["page/section"], +summary:"reusable source content",evidence_refs:["supplied id"]}], +requirement_checks:[{id:"R1",status:"met|not_met|uncertain",evidence_refs:["supplied id"]}], +artifacts:[{target_path:"actual final path",working_path:"actual working path",save_state:"modified|saved", +evidence_refs:["supplied id"]}],invalidated_work_item_ids:[]}. +Use [] when no update is established. Reference only the supplied evidence IDs. Preserve original +requirements and source facts. Extract essential facts NOW from real observed material/CLI output so +later stages do not reread it; unobserved contents remain unknown. Preparation uses delivery_state=none. +Use the exact facts_needed key when observed information answers that key. Keep partial source coverage +in materials even if not all required facts have been acquired. Opening a reference image, a contact +sheet, an audit or a script +that reports failure is not delivery progress. A finished stage need not satisfy a whole requirement. +unit_met=true ONLY when the full fixed work unit is satisfied; for delivery it requires verified level +and concrete evidence. Saved requires actual persistence to the target, not a working copy or title. +File existence/compilation does not prove content or function. Do not invent a demand to repeat every +successful intermediate action. Mark unmet/uncertain criteria explicitly and preserve actual progress. +Later contradictions invalidate affected units; do not invalidate old work from an uncertain screenshot +alone. Do not read evaluators, hidden grading rules, answers or rewards. Be concise and evidence-based.""" + +PROMPT += ''' +scope=task means reusable SOURCE CONTENT, not whole-task completion. scope=stage is only transient UI +state. A confirmed paragraph or data field is task-scope even while the rest of its source is unread. +Not seeing a source again does not contradict its recorded facts. Retain confirmed source values; +change one only with actual contradictory evidence and a nonempty correction_reason on that fact. +Use the precise registered source key when it is fully answered; store partial sections as separate +reusable facts/materials. Do not mark an entire packet confirmed when required fields are missing. +remaining must describe THIS stage's unmet exit conditions, not a list of every later task obligation. +Only mark unit_met when the original full unit is satisfied. Historical registered evidence may support +source facts; stage completion must also cite at least one CURRENT evidence ID. +All top-level keys are mandatory, including invalidated_work_item_ids:[] when nothing is invalidated. +Empty-update shape: {"stage_outcome":"uncertain","unit_met":false,"delivery_state":"none", +"summary":"Explain the observed state","remaining":[],"evidence_refs":[],"facts":[],"materials":[], +"requirement_checks":[],"artifacts":[],"invalidated_work_item_ids":[]}. +Use actual observed evidence and updates, not this example's empty values. A stage purpose=prepare is +source acquisition only even when attached to a delivery unit; it cannot finish that delivery unit. +For lists/tables/options, include entries:{"stable item/row/position":"observed value"} and complete:false +until the requested coverage is actually observed. New entries extend the old collection; conflicting +entries or scalar changes require current evidence and correction_reason. Never label a partial list +as a complete registered dependency. Temporary UI state is not required source content. +This request contains selected evidence, not the whole execution log. output_omitted_chars marks an +explicit excerpt: do not claim to have checked omitted contents or a complete source packet. Extract +only supported fields; ask for the missing section rather than another full-document audit. +For prepare, answer fact_targets using their EXACT registered keys and record partial coverage; +unit_met=false, delivery_state=none, and do not assess unrelated delivery requirements. +For produce/save/verify, check only the current stage exit condition and its fixed delivery unit. +Record partial implementation/saving without demanding all future task units. Return only memory deltas, +not a rewrite of existing facts/materials, and no new optional research or inspection requirements. +Missing source fields do not invalidate an independently useful, correctly implemented partial stage. +If that bounded exit condition is met, report stage_outcome=met with unit_met=false and the observed +implemented/saved level. Keep the full unit incomplete and preserve its missing values for later work. +Incomplete prose facts may be appended across stages: complete:false preserves earlier sections without +claiming a complete source. Mark complete:true only when the exact registered field/section is fully answered. +Never replace a known inventory with the contents of a different document under the same scalar key; +prefer document/field-specific facts and preserve source_ref. Do not guess identity or ownership. +''' + + +def result_error(value): + """Precise contract failures; never invent evidence or coerce unsupported results.""" + for key, choices in {"stage_outcome": {"met", "not_met", "uncertain"}, + "delivery_state": {"none", "implemented", "saved", "verified"}}.items(): + if value.get(key) not in tuple(choices): + return key + " must be one of " + str(sorted(choices)) + for key, kind in {"unit_met": bool, "summary": str}.items(): + if type(value.get(key)) is not kind: + return key + " must be " + kind.__name__ + lists = ("remaining", "evidence_refs", "facts", "materials", "requirement_checks", "artifacts", + "invalidated_work_item_ids") + for key in lists: + if not isinstance(value.get(key), list): + return key + " must be an array (use [] for no updates)" + for key in ("remaining", "evidence_refs", "invalidated_work_item_ids"): + if any(not isinstance(x, str) for x in value[key]): + return key + " must contain only strings" + fields = {"facts": ("key", "value", "source_ref"), "materials": ("source_ref", "summary"), + "requirement_checks": ("id",), "artifacts": ("target_path", "working_path")} + enums = {"facts": {"status": {"confirmed", "uncertain"}, "scope": {"task", "stage"}}, + "requirement_checks": {"status": {"met", "not_met", "uncertain"}}, + "artifacts": {"save_state": {"modified", "saved"}}} + for collection, strings in fields.items(): + for index, record in enumerate(value[collection]): + path = f"{collection}[{index}]" + if not isinstance(record, dict): + return path + " must be an object" + for key in strings: + if not isinstance(record.get(key), str): + return path + "." + key + " must be a string" + arrays = ("evidence_refs", "covered_parts") if collection == "materials" else ("evidence_refs",) + for key in arrays: + if not isinstance(record.get(key), list) or any(not isinstance(x, str) for x in record[key]): + return path + "." + key + " must be an array of strings" + for key, choices in enums.get(collection, {}).items(): + if record.get(key) not in tuple(choices): + return path + "." + key + " must be one of " + str(sorted(choices)) + if collection == "facts": + if "complete" in record and type(record["complete"]) is not bool: + return path + ".complete must be a boolean" + if "entries" in record and (not isinstance(record["entries"], dict) or any( + not isinstance(k, str) or not isinstance(v, str) + for k, v in record["entries"].items())): + return path + ".entries must map strings to strings" + return "" + + +def valid_result(value): + return not result_error(value) + + +def _excerpt(text, limit): + if len(text) <= limit: + return text + return text[:limit // 2] + "\n[... omitted source content; not inspected ...]\n" + text[-limit // 2:] + + +def _actions(history, output_budget): + """Keep receipts and semantic deltas, not repeated snapshot IDs/ancestry/whole trees.""" + actions = [] + previous = {} + output_records = max(1, sum(bool(r.get("full_output_ref") or r.get("stdout") or r.get("stderr")) + for r in history)) + limit = max(1000, output_budget // output_records) + for record in history: + action = {k: record[k] for k in ("evidence_ref", "kind", "operation", "target", "returncode", + "observed_change", "relevant_change", "target_effect", + "full_output_ref") if k in record} + if record.get("kind") != "cli" and record.get("action"): + action["action"] = record["action"] + output = dict(record) + if record.get("full_output_ref"): + output.update(json.loads(Path(record["full_output_ref"]).read_text(encoding="utf-8"))) + omitted = {} + for key in ("stdout", "stderr"): + text = output.get(key, "") + if text: + action[key] = _excerpt(text, limit) + if len(text) > limit: + omitted[key] = len(text) - limit + if omitted: + action["output_omitted_chars"] = omitted + changes, current = [], {} + for control in record.get("observed_controls", []): + key = (control.get("role"), control.get("name"), control.get("x"), control.get("y")) + value = {k: control[k] for k in ("role", "name", "value", "focused", "checked", "selected") + if k in control} + current[key] = value + if value != previous.get(key): + changes.append(value) + previous = current + if changes: + action["control_changes"] = changes + actions.append(action) + return actions + + +def _failure_result(memory, history, end_reason, evidence): + """No semantic success claim from a known error or an entirely ineffective local attempt.""" + pure_no_effect = (history and all(h.get("kind") == "gui" + and not h.get("relevant_change", h.get("observed_change")) + for h in history)) + skip = (end_reason == "execution_error" or not history and end_reason != "stage_done_requested" + or pure_no_effect and memory.active_stage["purpose"] not in {"save", "verify"}) + if not skip: + return None + return {"stage_outcome": "uncertain", "unit_met": False, "delivery_state": "none", + "summary": "No semantic acceptance: " + end_reason, + "remaining": ["Inspect any partial effects before recovery; do not repeat the failed method."], + "evidence_refs": list(evidence)[-1:], "facts": [], "materials": [], + "requirement_checks": [], "artifacts": [], "invalidated_work_item_ids": [], + "check_mode": "local_failure"} + + +def verify_goal(client, instruction, screenshot, *, memory, history, end_reason, evidence): + local = _failure_result(memory, history, end_reason, evidence) + if local is not None: + return local + stage = memory.active_stage + work = memory.work_items[stage["work_item_id"]] + targets = stage.get("fact_targets", work["facts_needed"]) + keys = set(stage["input_fact_keys"]) | {f["key"] for f in targets} + budget = client.config.stage_verify_max_chars + actions = _actions(history, budget // 2) + context = {"task": instruction, + "stage": {k: v for k, v in stage.items() if k != "command"}, + "command_excerpt": _excerpt(stage.get("command", ""), 3000), + "fixed_work_item": {**work, "facts_needed": targets}, + "requirements": [{k: r[k] for k in ("id", "requirement", "kind")} + for r in memory.requirements if r["id"] in work["requirement_ids"] + or r["kind"] == "constraint"], + "known_facts": [f for f in memory.facts.values() if f["key"] in keys], + "end_reason": end_reason, "actual_actions": actions, "evidence_ids": list(evidence), + "historical_evidence_ids": sorted(memory.known_evidence)} + text = json.dumps(context, ensure_ascii=False) + if len(text) > budget: + # Receipt deltas stay intact. Explicitly excerpt source text, never silently claim full coverage. + for index, original in enumerate(context["known_facts"]): + fact = {k: v for k, v in original.items() if k != "entries"} + fact["value"] = _excerpt(fact["value"], max(1000, budget // (2 * max(1, len(keys))))) + if fact["value"] != original["value"]: + fact["value_excerpt_only"] = True + context["known_facts"][index] = fact + text = json.dumps(context, ensure_ascii=False) + content = [{"type": "text", "text": text}] + source_frames = [h["screenshot"] for h in history if h.get("source_frame")] + paths = source_frames[-5:] if stage["purpose"] == "prepare" else ( + [history[0]["screenshot"]] if history else []) + frames = [Path(p) for p in dict.fromkeys(paths + [str(screenshot)])] + for path in frames: + content.extend([{"type": "text", "text": "Observed screenshot: " + str(path)}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + + base64.b64encode(path.read_bytes()).decode("ascii")}}]) + messages = [{"role": "system", "content": PROMPT}, {"role": "user", "content": content}] + + def validate(value): + error = result_error(value) + if not error: + allowed = set(evidence) | memory.known_evidence + records = [value, *value["facts"], *value["materials"], + *value["artifacts"], *value["requirement_checks"]] + unknown = sorted({ref for r in records for ref in r["evidence_refs"] if ref not in allowed}) + if unknown: + error = "evidence_refs contain unregistered IDs: " + json.dumps(unknown) + elif value["stage_outcome"] == "met" and not set(value["evidence_refs"]) & set(evidence): + error = "a met stage must cite at least one CURRENT evidence ID" + if error: + messages.append({"role": "user", "content": + "Invalid stage verification JSON: " + error + ". Return the COMPLETE corrected object " + "with every mandatory key, without changing the observed facts or inventing evidence."}) + return not error + + result = client.json_request(messages, validate, role="verify_stage") + result.update(check_mode="source" if stage["purpose"] == "prepare" else "delivery", + context_chars=len(text), image_count=len(frames)) + return result diff --git a/src/tandem_eval/__main__.py b/src/tandem_eval/__main__.py index b4b30a8..8dad50e 100644 --- a/src/tandem_eval/__main__.py +++ b/src/tandem_eval/__main__.py @@ -36,7 +36,7 @@ def main(argv: list[str] | None = None) -> int: run_cmd.add_argument("--allow-cli", choices=["true", "false"], default=None, help="harness 是否允许 CLI 路由") run_cmd.add_argument("--route-policy", choices=["auto", "gui", "cli"], default=None) - run_cmd.add_argument("--execution-mode", choices=["step", "stage"], default=None) + run_cmd.add_argument("--execution-mode", choices=["step", "stage", "goal"], default=None) run_cmd.add_argument("--stage-max-actions", type=int, default=None, help="阶段内动作数上限,达到后请求剩余工作的重规划") run_cmd.add_argument("--slow-model", default=None, metavar="NAME", diff --git a/src/tandem_eval/config_schema.py b/src/tandem_eval/config_schema.py index e6d9ceb..bb9ae28 100644 --- a/src/tandem_eval/config_schema.py +++ b/src/tandem_eval/config_schema.py @@ -241,8 +241,8 @@ def validate(config: dict) -> list[str]: if not isinstance(execution, dict): problems.append("harness.execution(应为对象)") else: - if execution.get("mode", "step") not in {"step", "stage"}: - problems.append("harness.execution.mode(应为 step / stage)") + if execution.get("mode", "step") not in {"step", "stage", "goal"}: + problems.append("harness.execution.mode(应为 step / stage / goal)") limit = execution.get("stage_max_actions", 24) if type(limit) is not int or limit < 1: problems.append("harness.execution.stage_max_actions(应为正整数)") diff --git a/src/tandem_eval/entry.py b/src/tandem_eval/entry.py index 7406ec7..df4150f 100644 --- a/src/tandem_eval/entry.py +++ b/src/tandem_eval/entry.py @@ -103,8 +103,14 @@ def __init__(self, *, stage_mode=False): self.stage_mode = stage_mode def __call__(self, instruction, screenshot, targets, recent, tree, - verified_focus, requirements, remaining_steps): + verified_focus=None, requirements=None, remaining_steps=None, *, memory=None): self.calls += 1 + if memory is not None and self.calls == 1: + return {"done": False, "work_item_id": memory.ready_items()[0]["id"], + "route": "gui", "purpose": "produce", "goal": "mock click", + "completion_criteria": "mock receipt", "method": "mock click", + "inputs": [], "input_fact_keys": [], "guidance": "", "max_actions": 1, + "command": ""} if self.calls == 1 and targets: target = targets[0] plan = { @@ -146,19 +152,32 @@ def rank(self, **kwargs): ) -def mock_components(*, stage_mode=False) -> Components: +def mock_components(*, stage_mode=False, goal_mode=False) -> Components: from tandem.gui.candidates import ActionCandidateBuilder from tandem.gui.compile import gui_action from tandem.gui.proposals import focused_candidates from tandem.gui.semantics import enrich_targets from tandem.gui.targets import A11yTargetBuilder + from tandem.memory.state import TaskMemory + + def goal_verifier(*args, **kwargs): + return {"stage_outcome": "met", "unit_met": True, "delivery_state": "verified", + "summary": "mock evidence only", "remaining": [], + "evidence_refs": list(kwargs["evidence"]), "facts": [], "materials": [], + "requirement_checks": [], "artifacts": [], "invalidated_work_item_ids": []} + + def requirements(instruction): + return [{"id": "R1", "requirement": instruction, "kind": "delivery", "units": [instruction], + "unit_fact_keys": [[]], "facts_needed": [], "output_path": None}] if goal_mode else [] return Components( planner=_ScriptedPlanner(stage_mode=stage_mode), verifier=_ScriptedVerifier(), - requirements=lambda instruction: [], selector=_FirstCandidateSelector(), + requirements=requirements, selector=_FirstCandidateSelector(), target_builder=A11yTargetBuilder(), candidate_builder=ActionCandidateBuilder(), semantic_enricher=enrich_targets, focus_candidates=focused_candidates, grounder=None, compile_gui=gui_action, + memory_factory=TaskMemory if goal_mode else None, + stage_verifier=goal_verifier if goal_mode else None, ) @@ -166,7 +185,8 @@ def build_session(spec: dict, *, llm_tap: LlmTap | None = None) -> TandemSession config = SessionConfig(**spec.get("harness_config", {})) benchmark = spec.get("benchmark") if benchmark == "mock": - return TandemSession(mock_components(stage_mode=config.execution_mode == "stage"), config=config) + return TandemSession(mock_components(stage_mode=config.execution_mode == "stage", + goal_mode=config.execution_mode == "goal"), config=config) if benchmark in {"osworld1", "osworld2", "osworld21"}: from tandem.components import build_default from tandem.models.slow import SlowModelClient diff --git a/src/tandem_eval/env.py b/src/tandem_eval/env.py index 38f5c5f..77b2bd3 100644 --- a/src/tandem_eval/env.py +++ b/src/tandem_eval/env.py @@ -7,10 +7,13 @@ from __future__ import annotations import hashlib +import io from dataclasses import dataclass, field from pathlib import Path from typing import Protocol +from PIL import Image + from tandem.protocol.models import ( AccessibilityTree, Action, @@ -30,6 +33,8 @@ def reset(self, task: dict) -> Observation: ... def execute(self, action: Action, *, budget: Budget, task: dict) -> tuple[Receipt, Observation]: ... + def observe(self, task: dict, *, budget: Budget, previous: Observation) -> Observation: ... + def fetch_task(self, task_id: str) -> dict: ... def export_outputs(self, paths: list[str], episode_dir) -> list[dict]: ... @@ -75,7 +80,9 @@ def fetch_task(self, task_id: str, domain: str = "") -> dict: def _screenshot_ref(self, task_id: str, step_idx: int) -> tuple[str, str]: key = f"{task_id}:{step_idx}" if key not in self.shots: - self.shots[key] = f"png-{key}".encode() + image = io.BytesIO() + Image.new("RGB", (1920, 1080), "white").save(image, format="PNG") + self.shots[key] = image.getvalue() digest = hashlib.sha256(self.shots[key]).hexdigest() if self.base_dir is not None: target = self.base_dir / "screenshots" / f"step_{step_idx:02d}.png" @@ -140,6 +147,9 @@ def execute(self, action: Action, *, budget: Budget, task: dict) -> tuple[Receip self.screenshot_counter += 1 return receipt, self._observation(task, step_idx, budget, receipt) + def observe(self, task, *, budget, previous): + return self._observation(task, previous.step_idx, budget, previous.last_receipt) + def export_outputs(self, paths: list[str], episode_dir) -> list[dict]: records = [] for index, path in enumerate(paths): diff --git a/src/tandem_eval/osworld.py b/src/tandem_eval/osworld.py index f8d3905..19685b8 100644 --- a/src/tandem_eval/osworld.py +++ b/src/tandem_eval/osworld.py @@ -47,8 +47,9 @@ def png_size(data: bytes) -> tuple[int, int]: def gui_code(action: GuiAction) -> str: params = action.parameters - if action.action_type in {"click", "double_click"}: - method = "click" if action.action_type == "click" else "doubleClick" + if action.action_type in {"click", "double_click", "right_click"}: + method = {"click": "click", "double_click": "doubleClick", "right_click": "rightClick"}[ + action.action_type] return f"pyautogui.{method}(x={params['x']}, y={params['y']})" if action.action_type == "hotkey": steps = [] @@ -66,7 +67,12 @@ def gui_code(action: GuiAction) -> str: steps.append("pyautogui.press('f2')") if params.get("input_mode", "replace") == "replace": steps.append("pyautogui.hotkey('ctrl', 'a')") - steps.append(f"pyautogui.write({params['text']!r})") + if params["text"].isascii(): + steps.append(f"pyautogui.write({params['text']!r})") + else: + steps.append("__import__('subprocess').run(['xclip','-selection','clipboard','-t','UTF8_STRING']," + f"input={params['text']!r}.encode('utf-8'),check=True)") + steps.append("pyautogui.hotkey('ctrl','v')") return "; ".join(steps) raise ValueError(f"不支持的 GUI 动作类型: {action.action_type}") @@ -126,6 +132,7 @@ def _bootstrap_runtime(self) -> None: os.environ["OSWORLD_FILE_BASE_URL"] = str(self.asset_root) self.env: Any = None self._step_seq = 0 + self._frame_seq = 0 self.last_screenshot: bytes = b"" self.last_a11y: str = "" self.last_browser_snapshot: dict | None = None @@ -182,13 +189,15 @@ def _observation(self, task: dict, step_idx: int, budget: Budget, receipt: Receipt | None) -> Observation: stage = self.run_root / "obs_stage" / str(task["id"]) stage.mkdir(parents=True, exist_ok=True) - shot_path = stage / f"step_{step_idx:02d}.png" + self._frame_seq += 1 + frame_name = f"step_{step_idx:02d}_obs_{self._frame_seq:04d}" + shot_path = stage / f"{frame_name}.png" shot_path.write_bytes(self.last_screenshot) a11y_path: str | None = None digest = node_count = None present = bool(self.last_a11y.strip()) if present: - a11y_path = str(stage / f"step_{step_idx:02d}.xml") + a11y_path = str(stage / f"{frame_name}.xml") Path(a11y_path).write_text(self.last_a11y, encoding="utf-8") digest = hashlib.sha256(self.last_a11y.encode()).hexdigest() node_count = self.last_a11y.count("<") @@ -196,7 +205,7 @@ def _observation(self, task: dict, step_idx: int, budget: Budget, browser_snapshot = None if self.last_browser_snapshot is not None: serialized = json.dumps(self.last_browser_snapshot, ensure_ascii=False) - browser_path = stage / f"step_{step_idx:02d}.dom.json" + browser_path = stage / f"{frame_name}.dom.json" browser_path.write_text(serialized, encoding="utf-8") browser_snapshot = BrowserSnapshot( present=True, @@ -229,6 +238,11 @@ def _capture(self) -> None: public = {key: value for key, value in diagnostic.items() if key != "url"} print(json.dumps({"event": "cdp_snapshot", **public}, ensure_ascii=False), flush=True) + def observe(self, task, *, budget, previous): + """Refresh screenshot, A11y and CDP without executing or replaying any action.""" + self._capture() + return self._observation(task, previous.step_idx, budget, previous.last_receipt) + # ------------------------------------------------------------------ 协议 def _seed_setup_cache(self, task_id: str) -> None: """把共享盘已缓存的 setup 文件播种进本次 evaluation_cache(旧 runtime.py:532-539), @@ -291,9 +305,17 @@ def acknowledged(command): stdout = result.get("output", "") stderr = result.get("error", "") returncode = int(result.get("returncode", 0)) + output_ref = None + if kind == "cli": + output = self.run_root / "cli_outputs" / f"action_{self._step_seq:04d}.json" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps({"stdout": stdout, "stderr": stderr, + "returncode": returncode}), encoding="utf-8") + output_ref = str(output) return Receipt( returncode=returncode, stdout=stdout[-RECEIPT_TAIL:], stderr=stderr[-RECEIPT_TAIL:], kind=kind if kind in {"cli", "gui", "wait"} else "gui", acknowledged=True, + full_output_ref=output_ref, ) def execute(self, action: Action, *, budget: Budget, task: dict) -> tuple[Receipt, Observation]: diff --git a/src/tandem_eval/recorder.py b/src/tandem_eval/recorder.py index fe26c6c..2df4145 100644 --- a/src/tandem_eval/recorder.py +++ b/src/tandem_eval/recorder.py @@ -57,6 +57,10 @@ def __init__(self, episode_dir: Path, task: dict, *, a11y_full: bool = False): self.dir.mkdir(parents=True, exist_ok=True) self.task = task self.a11y_full = a11y_full + self.executed_steps = 0 + self.environment_seconds = {} + self.model_usage = {} + self.fast_model = {"choice_cycles": 0, "seconds": 0.0, "cycles_with_image": 0} self.started = time.perf_counter() self._traj = (self.dir / "traj.jsonl").open("w", encoding="utf-8") self._events = (self.dir / "events.jsonl").open("w", encoding="utf-8") @@ -65,6 +69,7 @@ def __init__(self, episode_dir: Path, task: dict, *, a11y_full: bool = False): # ------------------------------------------------------------------ 逐步记录 def traj(self, observation: Observation, action: Action, receipt: Receipt | None) -> None: + self.executed_steps += 1 browser_snapshot = observation.browser_snapshot browser_record = browser_snapshot.to_dict() if browser_snapshot else None if browser_snapshot and browser_snapshot.present and browser_snapshot.path: @@ -89,9 +94,40 @@ def traj(self, observation: Observation, action: Action, receipt: Receipt | None self._traj.flush() def event(self, event: dict) -> None: + if event.get("route") == "environment_timing": + phase = event["phase"] + self.environment_seconds[phase] = self.environment_seconds.get(phase, 0.0) + event["seconds"] + if event.get("route") == "gui" and event.get("ranking"): + ranking = event["ranking"] + self.fast_model["choice_cycles"] += 1 + self.fast_model["seconds"] += ranking["latency_ms"] / 1000 + self.fast_model["cycles_with_image"] += int(ranking["used_image"]) + if event.get("route") == "stage_verified": + event = self._persist_evidence(event) self._events.write(json.dumps(event, ensure_ascii=False) + "\n") self._events.flush() + def task_memory(self, state: dict) -> None: + """Persist the latest host memory and retain its transient evidence before cleanup.""" + _write_json(self.dir / "task_memory.json", self._persist_evidence(state)) + + def _persist_evidence(self, state): + def persist(value): + if isinstance(value, dict): + return {k: persist(v) for k, v in value.items()} + if isinstance(value, list): + return [persist(v) for v in value] + if (isinstance(value, str) and any(part in value for part in ("/obs_stage/", "/cli_outputs/")) + and Path(value).is_file()): + target = self.dir / "evidence" / Path(value).name + target.parent.mkdir(exist_ok=True) + if not target.exists(): + shutil.copy2(value, target) + return str(target) + return value + + return persist(state) + def record_llm_call(self, *, role: str, request: Any, response: Any, **meta: Any) -> None: """慢模型客户端的 llm_sink;全量请求/响应落盘,密钥脱敏。""" self._llm.write( @@ -110,6 +146,16 @@ def record_llm_call(self, *, role: str, request: Any, response: Any, **meta: Any + "\n" ) self._llm.flush() + stats = self.model_usage.setdefault(role, {"calls": 0, "seconds": 0.0, "prompt_tokens": 0, + "completion_tokens": 0, "reasoning_tokens": 0, "cached_tokens": 0, "calls_missing_usage": 0}) + stats["calls"] += 1 + stats["seconds"] += meta.get("elapsed_seconds", 0.0) + usage = response.get("usage") or {} + stats["calls_missing_usage"] += int(not usage) + for key in ("prompt_tokens", "completion_tokens"): + stats[key] += usage.get(key, 0) + stats["reasoning_tokens"] += (usage.get("completion_tokens_details") or {}).get("reasoning_tokens", 0) + stats["cached_tokens"] += (usage.get("prompt_tokens_details") or {}).get("cached_tokens", 0) def screenshot(self, step_idx: int, data: bytes) -> None: target = self.dir / "screenshots" / f"step_{step_idx:02d}.png" @@ -150,7 +196,7 @@ def finalize( "task_id": self.task["id"], "harness": f"tandem {__version__}", "stop_reason": stop_reason, - "steps": steps, + "steps": self.executed_steps if exception is not None else steps, "status": status, "score": score, "exception_type": type(exception).__name__ if exception else None, @@ -163,6 +209,9 @@ def finalize( self.dir / "timing.json", { "wall_seconds": round(time.perf_counter() - self.started, 3), + "environment_seconds": self.environment_seconds, + "model_usage_by_role": self.model_usage, + "fast_model": self.fast_model, "status": status, }, ) diff --git a/src/tandem_eval/rjob_spec.py b/src/tandem_eval/rjob_spec.py index 99250ba..88f4e08 100644 --- a/src/tandem_eval/rjob_spec.py +++ b/src/tandem_eval/rjob_spec.py @@ -117,6 +117,11 @@ def _harness_config_section(config: dict, slow_api: dict, fast_api: dict) -> dic "allow_cli": bool(route.get("allow_cli", True)), "cli_timeout_seconds": int(route.get("cli_timeout_seconds", 90)), "slow_max_tokens": int(slow_api.get("max_tokens", 16384)), + "slow_verify_max_tokens": int(execution.get("verify_max_tokens", 16384)), + "slow_verify_reasoning_effort": execution.get("verify_reasoning_effort"), + "stage_verify_max_chars": int(execution.get("verify_max_chars", 48000)), + "slow_image_quality": int(execution.get("image_quality", 85)), + "slow_max_request_bytes": int(execution.get("max_request_bytes", 2 * 1024 * 1024)), "slow_timeout_seconds": float(slow_api.get("timeout_seconds", 120)), "slow_reasoning_effort": slow_api.get("reasoning_effort"), "slow_direct_api": bool(slow_api.get("direct_api", False)), @@ -193,7 +198,8 @@ def build_job_spec(*, config: dict, run_id: str, repo_root: Path, "image_stage": _image_stage_section(config), "harness": { "name": "tandem", - "mode": "stage" if harness_config["execution_mode"] == "stage" else "planner_led", + "mode": (harness_config["execution_mode"] + if harness_config["execution_mode"] in {"stage", "goal"} else "planner_led"), "models": {"slow": slow_name, "fast": fast_name}, }, "harness_config": harness_config, diff --git a/src/tandem_eval/runner.py b/src/tandem_eval/runner.py index 8a9d18c..b84320c 100644 --- a/src/tandem_eval/runner.py +++ b/src/tandem_eval/runner.py @@ -8,6 +8,7 @@ from pathlib import Path from tandem.protocol.models import Observation +from tandem.runtime.goal_stage import ObservationRequired from tandem.runtime.session import TandemSession from .env import budget_of @@ -25,7 +26,16 @@ def run_episode(env, session: TandemSession, recorder: EpisodeRecorder, task: di started = time.perf_counter() steps_used = 0 stop_reason = "max_steps" - observation: Observation = env.reset(task) + + def environment_call(phase, function, *args, **kwargs): + before = time.perf_counter() + try: + return function(*args, **kwargs) + finally: + recorder.event({"route": "environment_timing", "phase": phase, + "seconds": time.perf_counter() - before}) + + observation: Observation = environment_call("setup", env.reset, task) outputs: list[dict] = [] events_start = len(session.events) # 本题事件起点:act 后增量落盘(planner/快模型/验证) @@ -34,6 +44,8 @@ def drain_events() -> None: for event in session.events[events_start:]: recorder.event(event) events_start = len(session.events) + if session.goal is not None: + recorder.task_memory(session.goal.memory.snapshot()) while True: budget = budget_of( @@ -50,19 +62,27 @@ def drain_events() -> None: stop_reason = "wall_time_limit" break observation = replace(observation, budget=budget) - action = session.act(observation) - drain_events() + try: + action = session.act(observation) + except ObservationRequired as refresh: + recorder.event({"route": "observation_refresh", "step": steps_used, + "reason": str(refresh), "action_replayed": False}) + observation = environment_call("observe", env.observe, task, budget=budget, previous=observation) + continue + finally: + drain_events() recorder.event({"step": steps_used, "kind": "action", "action": action.to_dict()}) if getattr(action, "kind", "") in TERMINAL_KINDS: stop_reason = "done" if action.kind == "finish" else action.reason # 终态仍须送达 VM(DONE/FAIL 触发评分钩子,旧 runtime 同款) - env.execute(action, budget=budget, task=task) + environment_call("execute", env.execute, action, budget=budget, task=task) if action.kind == "finish": - outputs = env.export_outputs( + outputs = environment_call( + "export", env.export_outputs, list(action.outputs) + list(extra_exports or []), recorder.dir ) break - receipt, observation = env.execute(action, budget=budget, task=task) + receipt, observation = environment_call("execute", env.execute, action, budget=budget, task=task) recorder.traj(observation, action, receipt) # 环境提供素材属性时写入 episode 目录(真实环境 M3 同样暴露 last_* 属性) shot = getattr(env, "last_screenshot", None) @@ -72,7 +92,9 @@ def drain_events() -> None: if a11y_text: recorder.a11y(steps_used, a11y_text) steps_used += 1 - raw = env.score(task) + raw = environment_call("evaluate", env.score, task) + if session.goal is not None: + recorder.task_memory(session.goal.memory.snapshot()) score = official_score(raw) if raw is not None else None return recorder.finalize( stop_reason=stop_reason, @@ -129,6 +151,8 @@ def run(spec: dict, *, env, session: TandemSession, tasks: list[TaskRef], ) ) finally: + if recorder is not None and session.goal is not None: + recorder.task_memory(session.goal.memory.snapshot()) if llm_tap is not None: llm_tap.current = None # obs_stage 是 episode 内工作区(截图正本已由 recorder 写入 episode 目录, diff --git a/tests/eval/test_experiment_script.py b/tests/eval/test_experiment_script.py index 9c2d3ea..9773b32 100644 --- a/tests/eval/test_experiment_script.py +++ b/tests/eval/test_experiment_script.py @@ -54,6 +54,16 @@ def test_invalid_experiment_name_is_rejected(tmp_path): assert not (tmp_path / "bad name").exists() +def test_dry_run_selects_goal_mode_without_changing_script_default(tmp_path): + result = subprocess.run([ + "bash", str(SCRIPT), "--experiment", "goal-smoke", "--tasks", "008", "--execution-mode", "goal", + "--stage-max-actions", "2", "--dry-run"], capture_output=True, text=True, + env={**os.environ, "TANDEM_EXPERIMENT_ROOT": str(tmp_path)}) + assert result.returncode == 0, result.stderr + manifest = json.loads((tmp_path / "goal-smoke/manifest.json").read_text()) + assert manifest["execution_mode"] == "goal" and manifest["stage_max_actions"] == 2 + + def test_dry_run_uses_example_when_private_config_is_missing(tmp_path): env = { **os.environ, diff --git a/tests/eval/test_mock_chain.py b/tests/eval/test_mock_chain.py index 5391883..d7312b5 100644 --- a/tests/eval/test_mock_chain.py +++ b/tests/eval/test_mock_chain.py @@ -41,9 +41,10 @@ def _spec(tmp_path: Path, **overrides) -> dict: return base -@pytest.mark.parametrize("mode", ["step", "stage"]) +@pytest.mark.parametrize("mode", ["step", "stage", "goal"]) def test_full_mock_chain_produces_renderable_result_dir(tmp_path, mode): - episodes = run_from_spec(_spec(tmp_path, harness_config={"execution_mode": mode})) + episodes = run_from_spec(_spec(tmp_path, harness_config={"execution_mode": mode}, + requirements=None if mode == "goal" else [])) assert episodes and episodes[0]["status"] == "scored" run_root = tmp_path / "run" episode_dir = run_root / "episodes" / "mock-mock-000" @@ -64,6 +65,12 @@ def test_full_mock_chain_produces_renderable_result_dir(tmp_path, mode): assert episode["stop_reason"] == "done" and episode["score"] == 1.0 summary = json.loads((run_root / "summary.json").read_text()) assert summary["scored"] == 1 and summary["average_score"] == 1.0 + if mode == "goal": + memory = json.loads((episode_dir / "task_memory.json").read_text()) + assert memory["work_items"][0]["status"] == "done" + events = [json.loads(line) for line in (episode_dir / "events.jsonl").read_text().splitlines()] + assert any(e.get("route") == "stage_verified" for e in events) + assert any(e.get("route") == "observation_refresh" for e in events) # 渲染(demo 占位)可离线打开 page = render_summary.render(run_root) diff --git a/tests/eval/test_osworld_env.py b/tests/eval/test_osworld_env.py index 10fe121..98e516c 100644 --- a/tests/eval/test_osworld_env.py +++ b/tests/eval/test_osworld_env.py @@ -62,6 +62,30 @@ def test_png_size_parses_ihdr(): assert png_size(_png(1920, 1080)) == (1920, 1080) +def test_read_only_refresh_keeps_cdp_receipt_and_prior_evidence(tmp_path, env, monkeypatch): + snapshot = {"elements": [{"index": 1, "role": "entry", "name": "Receipt Date"}]} + monkeypatch.setattr("tandem_eval.osworld.capture_browser_snapshot", + lambda controller: (snapshot, {"status": "ok"})) + task = _task_file(tmp_path) + first = env.reset(task) + budget = Budget(steps_remaining=10, wall_seconds_remaining=60) + receipt, previous = env.execute(GuiAction("click", {"x": 1, "y": 2}), budget=budget, task=task) + steps, commands = list(env.env.steps), list(env.env.controller.commands) + before = Path(previous.screenshot.path).read_bytes() + refreshed = env.observe(task, budget=budget, previous=previous) + assert refreshed.protocol_version == "0.2" and refreshed.last_receipt is receipt + assert refreshed.step_idx == previous.step_idx + assert env.env.steps == steps and env.env.controller.commands == commands + assert refreshed.browser_snapshot.source == "cdp" + assert json.loads(Path(refreshed.browser_snapshot.path).read_text()) == snapshot + assert refreshed.screenshot.path != previous.screenshot.path != first.screenshot.path + assert Path(previous.screenshot.path).read_bytes() == before + + +def test_right_click_codegen_uses_real_mouse_button(): + assert gui_code(GuiAction("right_click", {"x": 20, "y": 30})) == "pyautogui.rightClick(x=20, y=30)" + + # ------------------------------------------------------------------ adapter class FakeController: def __init__(self): diff --git a/tests/harness/conftest.py b/tests/harness/conftest.py index f7317ab..e91bc14 100644 --- a/tests/harness/conftest.py +++ b/tests/harness/conftest.py @@ -7,6 +7,7 @@ from pathlib import Path import pytest +from PIL import Image from tandem.components import Components from tandem.gui.candidates import ActionCandidateBuilder @@ -61,7 +62,7 @@ def make_observation(tmp_path: Path, step_idx: int, *, receipt: Receipt | None = tree_xml: str | None = A11Y_XML, steps: int = 50, wall: float = 1800.0, screenshot_sha: str = SHA) -> Observation: shot = tmp_path / f"step_{step_idx:02d}.png" - shot.write_bytes(b"png-bytes") + Image.new("RGB", (64, 64), "white").save(shot) a11y_path = None if tree_xml is not None: a11y_path = str(tmp_path / f"a11y_{step_idx:02d}.xml") diff --git a/tests/harness/test_goal_execution.py b/tests/harness/test_goal_execution.py new file mode 100644 index 0000000..1e94840 --- /dev/null +++ b/tests/harness/test_goal_execution.py @@ -0,0 +1,201 @@ +"""Synthetic regressions derived from real failure patterns; not benchmark acceptance.""" + +from unittest.mock import Mock + +import pytest + +from tandem.gui.compile import gui_action +from tandem.gui.goal_candidates import foreground_tree, goal_candidates +from tandem.gui.targets import A11yTargetBuilder +from tandem.memory.context import planning_context +from tandem.models.ranking import Ranking +from tandem.planning.goal_planner import goal_error, plan_goal +from tandem.runtime.goal_stage import ( + ObservationRequired, + _action_key, + _cycle_actions, + _loop_state, + _target_effect, +) +from tandem.verification.goal_verifier import result_error + +from .conftest import A11Y_XML, build_space, build_targets, gui_receipt, make_observation +from .test_goal_memory import fact, session +from .test_goal_stage import plan, requirements, result + + +def _space(tree, inputs=()): + tree = foreground_tree(tree) + targets = A11yTargetBuilder().build(tree, "test", 128, True, True, filter_occluded=True) + return targets, goal_candidates(build_space(targets), targets, tree, inputs) + + +def test_receipt_path_detects_a_b_a_b_without_manually_seeding_attempts(tmp_path): + s = session() + s.begin("Edit", requirements()) + s.goal.memory.start(plan(), 0) + chosen = [] + + def rank(**kw): + candidates = kw["action_space"].candidates() + toggles = [c for c in candidates if c.operation == "click" and "启用" in c.description] + if len(chosen) == 4: + assert not toggles + assert kw["action_space"].controls["return_to_planner"] + winner = next(c for c in candidates if c.id == "KEYPRESS_ESCAPE") + else: + winner = toggles[0] + chosen.append(winner) + return Ranking(winner, [], [], 0, 0, 1, True) + + s.components.selector.rank = rank + for step in range(5): + tree = A11Y_XML.replace('checked="false"', f'checked="{str(step % 2 == 1).lower()}"') + s.act(make_observation(tmp_path, step, tree_xml=tree, + receipt=gui_receipt() if step else None)) + assert len(s.goal.failed_actions) == 1 # Block current state; the other state blocks on return. + assert s.goal.memory.execution_constraints + assert planning_context(s.goal.memory)["execution_constraints"] + assert s.goal.attempts[0]["state"] == s.goal.attempts[2]["state"] + + +def test_duplicate_popup_nodes_do_not_remove_their_own_close_button(): + popup = ('' + '' + '') + tree = ('' + '' + + popup * 2 + '') + targets = A11yTargetBuilder().build(tree, "test", filter_occluded=True) + assert [t.name for t in targets].count("Close") == 2 + assert "PDF" not in [t.name for t in targets] + assert "Not covered" in [t.name for t in targets] + + +def test_explicit_popup_menu_excludes_the_covered_media_button(): + tree = ('' + '' + '' + '') + ts, space = _space(tree) + assert "More" not in [t.name for t in ts] + assert any("Exit picture-in-picture" in c.description for c in space.target_heads["click"]) + + +def test_browser_memory_and_playback_badges_do_not_create_new_loop_states(): + def state(name): + tree = ('') + return _loop_state(build_targets(tree), tree) + + assert state("Video - Memory usage - 47.1 MB") == state( + "Video – Audio playing - Memory usage - 108 MB") + assert state("Different page - Memory usage - 108 MB") != state("Video - Memory usage - 47.1 MB") + + +def test_four_state_cycle_is_detected_but_continuous_scroll_is_not(): + states = ["closed", "menu", "pip", "pip-menu"] + attempts = [{"work": "w", "state": state, "key": ("click", "menu", "")} + for state in states * 2] + assert _cycle_actions(attempts, "closed") == {("click", "menu", "")} + assert not _cycle_actions([{"state": "page", "key": ("scroll", "down", "")}] * 8, "page") + + +def test_overview_excludes_stale_app_and_keeps_unnamed_search_then_typing(): + tree = ('' + '