From 98f5623af6c08b8d5d26a93cd5514bec4c21e489 Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Mon, 20 Jul 2026 16:34:31 -0700 Subject: [PATCH 01/15] Expose Operator action trace timestamps in SDK --- .../narada-core/src/narada_core/models.py | 2 + .../src/narada_core/tracing/model.py | 20 +++++++ packages/narada/tests/test_action_trace.py | 53 +++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 packages/narada/tests/test_action_trace.py diff --git a/packages/narada-core/src/narada_core/models.py b/packages/narada-core/src/narada_core/models.py index 73095c6..a54ace8 100644 --- a/packages/narada-core/src/narada_core/models.py +++ b/packages/narada-core/src/narada_core/models.py @@ -96,6 +96,8 @@ class RemoteDispatchChatHistoryItem(TypedDict): class OperatorActionTraceItem(TypedDict): url: str action: str + startTs: NotRequired[int] + endTs: NotRequired[int] class GoToUrlTrace(TypedDict): diff --git a/packages/narada-core/src/narada_core/tracing/model.py b/packages/narada-core/src/narada_core/tracing/model.py index f655cf8..9ef9295 100644 --- a/packages/narada-core/src/narada_core/tracing/model.py +++ b/packages/narada-core/src/narada_core/tracing/model.py @@ -4,6 +4,7 @@ from pydantic import ( BaseModel, + ConfigDict, Field, NonNegativeInt, TypeAdapter, @@ -27,8 +28,27 @@ def _normalize_agent_type(agent_type: object) -> str: class OperatorActionTraceItem(BaseModel): + model_config = ConfigDict(populate_by_name=True) + url: str action: str + start_ts: NonNegativeInt | None = Field(default=None, alias="startTs") + end_ts: NonNegativeInt | None = Field(default=None, alias="endTs") + + @model_validator(mode="after") + def _check_timestamp_pair(self) -> OperatorActionTraceItem: + if (self.start_ts is None) != (self.end_ts is None): + raise ValueError("Operator action timestamps must be provided together") + if ( + self.start_ts is not None + and self.end_ts is not None + and self.end_ts < self.start_ts + ): + raise ValueError( + f"OperatorActionTraceItem: endTs ({self.end_ts}) must be >= " + f"startTs ({self.start_ts})" + ) + return self class GoToUrlTrace(BaseModel): diff --git a/packages/narada/tests/test_action_trace.py b/packages/narada/tests/test_action_trace.py new file mode 100644 index 0000000..124d3bb --- /dev/null +++ b/packages/narada/tests/test_action_trace.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import pytest +from narada_core.tracing.model import OperatorActionTraceItem, parse_action_trace +from pydantic import ValidationError + + +def test_parse_operator_action_trace_exposes_timestamps() -> None: + trace = parse_action_trace( + [ + { + "url": "https://example.com", + "action": "Clicked Submit", + "startTs": 1_000, + "endTs": 2_500, + } + ] + ) + + item = trace[0] + assert isinstance(item, OperatorActionTraceItem) + assert item.start_ts == 1_000 + assert item.end_ts == 2_500 + + +def test_parse_operator_action_trace_accepts_legacy_items_without_timestamps() -> None: + trace = parse_action_trace( + [{"url": "https://example.com", "action": "Clicked Submit"}] + ) + + item = trace[0] + assert isinstance(item, OperatorActionTraceItem) + assert item.start_ts is None + assert item.end_ts is None + + +@pytest.mark.parametrize( + "timestamps", + [ + {"startTs": 1_000}, + {"endTs": 2_000}, + {"startTs": 2_000, "endTs": 1_000}, + ], +) +def test_operator_action_trace_rejects_invalid_timestamp_ranges( + timestamps: dict[str, int], +) -> None: + with pytest.raises(ValidationError): + OperatorActionTraceItem( + url="https://example.com", + action="Clicked Submit", + **timestamps, + ) From f4c4b264058aa831ba1c287377c60909e28ec3f8 Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Mon, 20 Jul 2026 17:28:19 -0700 Subject: [PATCH 02/15] Require Operator action timestamps --- packages/narada-core/src/narada_core/models.py | 4 ++-- .../src/narada_core/tracing/model.py | 17 ++++------------- packages/narada/tests/test_action_trace.py | 11 ----------- 3 files changed, 6 insertions(+), 26 deletions(-) diff --git a/packages/narada-core/src/narada_core/models.py b/packages/narada-core/src/narada_core/models.py index a54ace8..38ef3c0 100644 --- a/packages/narada-core/src/narada_core/models.py +++ b/packages/narada-core/src/narada_core/models.py @@ -96,8 +96,8 @@ class RemoteDispatchChatHistoryItem(TypedDict): class OperatorActionTraceItem(TypedDict): url: str action: str - startTs: NotRequired[int] - endTs: NotRequired[int] + startTs: int + endTs: int class GoToUrlTrace(TypedDict): diff --git a/packages/narada-core/src/narada_core/tracing/model.py b/packages/narada-core/src/narada_core/tracing/model.py index 9ef9295..b52e6e1 100644 --- a/packages/narada-core/src/narada_core/tracing/model.py +++ b/packages/narada-core/src/narada_core/tracing/model.py @@ -4,7 +4,6 @@ from pydantic import ( BaseModel, - ConfigDict, Field, NonNegativeInt, TypeAdapter, @@ -28,22 +27,14 @@ def _normalize_agent_type(agent_type: object) -> str: class OperatorActionTraceItem(BaseModel): - model_config = ConfigDict(populate_by_name=True) - url: str action: str - start_ts: NonNegativeInt | None = Field(default=None, alias="startTs") - end_ts: NonNegativeInt | None = Field(default=None, alias="endTs") + start_ts: NonNegativeInt = Field(alias="startTs") + end_ts: NonNegativeInt = Field(alias="endTs") @model_validator(mode="after") - def _check_timestamp_pair(self) -> OperatorActionTraceItem: - if (self.start_ts is None) != (self.end_ts is None): - raise ValueError("Operator action timestamps must be provided together") - if ( - self.start_ts is not None - and self.end_ts is not None - and self.end_ts < self.start_ts - ): + def _check_timestamp_range(self) -> OperatorActionTraceItem: + if self.end_ts < self.start_ts: raise ValueError( f"OperatorActionTraceItem: endTs ({self.end_ts}) must be >= " f"startTs ({self.start_ts})" diff --git a/packages/narada/tests/test_action_trace.py b/packages/narada/tests/test_action_trace.py index 124d3bb..7843ae2 100644 --- a/packages/narada/tests/test_action_trace.py +++ b/packages/narada/tests/test_action_trace.py @@ -23,17 +23,6 @@ def test_parse_operator_action_trace_exposes_timestamps() -> None: assert item.end_ts == 2_500 -def test_parse_operator_action_trace_accepts_legacy_items_without_timestamps() -> None: - trace = parse_action_trace( - [{"url": "https://example.com", "action": "Clicked Submit"}] - ) - - item = trace[0] - assert isinstance(item, OperatorActionTraceItem) - assert item.start_ts is None - assert item.end_ts is None - - @pytest.mark.parametrize( "timestamps", [ From cbc317c0a7d8ec078f456c39672f27005ee868af Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Mon, 20 Jul 2026 17:36:34 -0700 Subject: [PATCH 03/15] Add dev action timing validation script --- .../validate_operator_action_trace_timing.py | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 scripts/validate_operator_action_trace_timing.py diff --git a/scripts/validate_operator_action_trace_timing.py b/scripts/validate_operator_action_trace_timing.py new file mode 100644 index 0000000..0afa319 --- /dev/null +++ b/scripts/validate_operator_action_trace_timing.py @@ -0,0 +1,154 @@ +"""Run an Operator task against dev and validate action-trace timestamps. + +Example: + + export NARADA_API_KEY="..." + uv run python scripts/validate_operator_action_trace_timing.py \ + --base-url "https:///fast/v2" + +The script refuses the production API endpoint, creates a temporary cloud +browser session, and always closes that session before exiting. +""" + +from __future__ import annotations + +import argparse +import asyncio +import os +import time +from collections.abc import Sequence +from datetime import UTC, datetime + +from narada import Agent, CloudBrowserEnvironment +from narada_core.tracing.model import OperatorActionTraceItem + +PRODUCTION_API_BASE_URL = "https://api.narada.ai/fast/v2" +DEFAULT_PROMPT = ( + "Go to https://example.com, read the page heading, and tell me the heading." +) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Validate Operator action-trace timestamps against a dev API.", + ) + parser.add_argument( + "--base-url", + default=os.getenv("NARADA_API_BASE_URL"), + help="Dev API base URL ending in /fast/v2. Defaults to NARADA_API_BASE_URL.", + ) + parser.add_argument( + "--prompt", + default=DEFAULT_PROMPT, + help="Operator task to run. It should require at least one browser action.", + ) + parser.add_argument( + "--timeout", + type=int, + default=600, + help="Agent timeout in seconds (default: 600).", + ) + args = parser.parse_args() + + if not args.base_url: + parser.error("Pass --base-url or set NARADA_API_BASE_URL.") + if args.base_url.rstrip("/") == PRODUCTION_API_BASE_URL: + parser.error("This dev test refuses to run against the production API.") + if "NARADA_API_KEY" not in os.environ: + parser.error("Set NARADA_API_KEY before running this script.") + if args.timeout <= 0: + parser.error("--timeout must be greater than zero.") + + return args + + +def _format_timestamp(timestamp_ms: int) -> str: + timestamp = datetime.fromtimestamp(timestamp_ms / 1000, tz=UTC) + return timestamp.isoformat(timespec="milliseconds") + + +def _validate_trace( + trace: Sequence[object] | None, +) -> list[OperatorActionTraceItem]: + if not trace: + raise AssertionError("Operator returned no action trace.") + + actions: list[OperatorActionTraceItem] = [] + for index, item in enumerate(trace): + if not isinstance(item, OperatorActionTraceItem): + raise AssertionError( + f"Action {index} has unexpected trace type {type(item).__name__}." + ) + actions.append(item) + + if len(actions) < 2: + raise AssertionError( + "Expected at least one browser action followed by the done action." + ) + + for index, action in enumerate(actions): + if action.end_ts < action.start_ts: + raise AssertionError( + f"Action {index} ends before it starts: " + f"{action.start_ts} > {action.end_ts}." + ) + if index > 0 and action.start_ts != actions[index - 1].end_ts: + raise AssertionError( + f"Action {index} is not contiguous: startTs={action.start_ts}, " + f"previous endTs={actions[index - 1].end_ts}." + ) + + if not actions[-1].action.startswith("Done:"): + raise AssertionError("The final trace item is not the done action.") + + return actions + + +async def _run(args: argparse.Namespace) -> None: + os.environ["NARADA_API_BASE_URL"] = args.base_url.rstrip("/") + + environment = CloudBrowserEnvironment( + session_name="operator-action-trace-timing-dev-test", + session_timeout=args.timeout + 120, + ) + agent = Agent(environment=environment) + run_started_ms = int(time.time() * 1000) + + try: + response = await agent.run(prompt=args.prompt, timeout=args.timeout) + run_finished_ms = int(time.time() * 1000) + actions = _validate_trace(response.action_trace) + + print(f"Request: {response.request_id}") + print(f"Response: {response.text}") + print() + print("Operator action timing:") + for index, action in enumerate(actions, start=1): + duration_ms = action.end_ts - action.start_ts + print( + f"{index:>2}. {action.action}\n" + f" start: {_format_timestamp(action.start_ts)} " + f"({action.start_ts})\n" + f" end: {_format_timestamp(action.end_ts)} " + f"({action.end_ts})\n" + f" duration: {duration_ms} ms" + ) + + total_duration_ms = actions[-1].end_ts - actions[0].start_ts + print() + print( + f"PASS: {len(actions)} contiguous actions; " + f"trace duration={total_duration_ms} ms; " + f"client elapsed={run_finished_ms - run_started_ms} ms." + ) + finally: + await environment.close(timeout=30) + + +def main() -> None: + args = _parse_args() + asyncio.run(_run(args)) + + +if __name__ == "__main__": + main() From 961b5e750b5b7b9ec36bdb5c16138f43a9078757 Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Mon, 20 Jul 2026 18:02:27 -0700 Subject: [PATCH 04/15] Run timing validation through local frontend --- .../validate_operator_action_trace_timing.py | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/scripts/validate_operator_action_trace_timing.py b/scripts/validate_operator_action_trace_timing.py index 0afa319..a4adf44 100644 --- a/scripts/validate_operator_action_trace_timing.py +++ b/scripts/validate_operator_action_trace_timing.py @@ -1,13 +1,12 @@ -"""Run an Operator task against dev and validate action-trace timestamps. +"""Run Operator through the local frontend and validate action-trace timestamps. Example: export NARADA_API_KEY="..." - uv run python scripts/validate_operator_action_trace_timing.py \ - --base-url "https:///fast/v2" + uv run python scripts/validate_operator_action_trace_timing.py -The script refuses the production API endpoint, creates a temporary cloud -browser session, and always closes that session before exiting. +The script uses the locally running caddie API, frontend, and development +extension. It launches a local Chrome window and closes that window on exit. """ from __future__ import annotations @@ -19,10 +18,13 @@ from collections.abc import Sequence from datetime import UTC, datetime -from narada import Agent, CloudBrowserEnvironment +from narada import Agent, BrowserConfig, BrowserEnvironment from narada_core.tracing.model import OperatorActionTraceItem PRODUCTION_API_BASE_URL = "https://api.narada.ai/fast/v2" +DEFAULT_LOCAL_API_BASE_URL = "http://127.0.0.1:8000/fast/v2" +DEFAULT_LOCAL_FRONTEND_URL = "http://localhost:3000" +DEFAULT_DEV_EXTENSION_ID = "ijdopnjleolkjakldkjplfhniiohnccf" DEFAULT_PROMPT = ( "Go to https://example.com, read the page heading, and tell me the heading." ) @@ -30,12 +32,22 @@ def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( - description="Validate Operator action-trace timestamps against a dev API.", + description="Validate Operator action-trace timestamps through the local stack.", ) parser.add_argument( "--base-url", - default=os.getenv("NARADA_API_BASE_URL"), - help="Dev API base URL ending in /fast/v2. Defaults to NARADA_API_BASE_URL.", + default=os.getenv("NARADA_API_BASE_URL", DEFAULT_LOCAL_API_BASE_URL), + help=(f"Local caddie API base URL (default: {DEFAULT_LOCAL_API_BASE_URL})."), + ) + parser.add_argument( + "--frontend-url", + default=DEFAULT_LOCAL_FRONTEND_URL, + help=f"Local frontend origin (default: {DEFAULT_LOCAL_FRONTEND_URL}).", + ) + parser.add_argument( + "--extension-id", + default=DEFAULT_DEV_EXTENSION_ID, + help=f"Installed local extension ID (default: {DEFAULT_DEV_EXTENSION_ID}).", ) parser.add_argument( "--prompt", @@ -50,10 +62,12 @@ def _parse_args() -> argparse.Namespace: ) args = parser.parse_args() - if not args.base_url: - parser.error("Pass --base-url or set NARADA_API_BASE_URL.") if args.base_url.rstrip("/") == PRODUCTION_API_BASE_URL: - parser.error("This dev test refuses to run against the production API.") + parser.error("This local test refuses to run against the production API.") + if not args.frontend_url: + parser.error("--frontend-url must not be empty.") + if not args.extension_id: + parser.error("--extension-id must not be empty.") if "NARADA_API_KEY" not in os.environ: parser.error("Set NARADA_API_KEY before running this script.") if args.timeout <= 0: @@ -107,9 +121,11 @@ def _validate_trace( async def _run(args: argparse.Namespace) -> None: os.environ["NARADA_API_BASE_URL"] = args.base_url.rstrip("/") - environment = CloudBrowserEnvironment( - session_name="operator-action-trace-timing-dev-test", - session_timeout=args.timeout + 120, + environment = BrowserEnvironment( + config=BrowserConfig( + initialization_url=f"{args.frontend_url.rstrip('/')}/initialize", + extension_id=args.extension_id, + ), ) agent = Agent(environment=environment) run_started_ms = int(time.time() * 1000) From c2a481310ab61c452fdf886b1acf776dc6d1a503 Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Mon, 20 Jul 2026 18:06:06 -0700 Subject: [PATCH 05/15] Make local timing validation rerunnable --- .../validate_operator_action_trace_timing.py | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/scripts/validate_operator_action_trace_timing.py b/scripts/validate_operator_action_trace_timing.py index a4adf44..c975cee 100644 --- a/scripts/validate_operator_action_trace_timing.py +++ b/scripts/validate_operator_action_trace_timing.py @@ -14,17 +14,22 @@ import argparse import asyncio import os +import signal +import socket import time from collections.abc import Sequence from datetime import UTC, datetime +from typing import NoReturn from narada import Agent, BrowserConfig, BrowserEnvironment from narada_core.tracing.model import OperatorActionTraceItem +from pydantic import ValidationError PRODUCTION_API_BASE_URL = "https://api.narada.ai/fast/v2" DEFAULT_LOCAL_API_BASE_URL = "http://127.0.0.1:8000/fast/v2" DEFAULT_LOCAL_FRONTEND_URL = "http://localhost:3000" DEFAULT_DEV_EXTENSION_ID = "ijdopnjleolkjakldkjplfhniiohnccf" +DEFAULT_CDP_PORT = 9222 DEFAULT_PROMPT = ( "Go to https://example.com, read the page heading, and tell me the heading." ) @@ -81,6 +86,30 @@ def _format_timestamp(timestamp_ms: int) -> str: return timestamp.isoformat(timespec="milliseconds") +def _is_cdp_browser_running() -> bool: + try: + with socket.create_connection(("127.0.0.1", DEFAULT_CDP_PORT), timeout=0.25): + return True + except OSError: + return False + + +def _raise_clear_timestamp_error(error: ValidationError) -> NoReturn: + missing_timestamps = any( + item["type"] == "missing" + and item["loc"] + and item["loc"][-1] in {"startTs", "endTs"} + for item in error.errors() + ) + if missing_timestamps: + raise RuntimeError( + "The local caddie process returned an action trace without timestamps. " + "Restart the backend after switching to " + "sp/operator-action-trace-timestamps, then run this script again." + ) from error + raise error + + def _validate_trace( trace: Sequence[object] | None, ) -> list[OperatorActionTraceItem]: @@ -121,17 +150,25 @@ def _validate_trace( async def _run(args: argparse.Namespace) -> None: os.environ["NARADA_API_BASE_URL"] = args.base_url.rstrip("/") + attach_to_existing = _is_cdp_browser_running() + if attach_to_existing: + print(f"Reusing the local Chrome process on CDP port {DEFAULT_CDP_PORT}.") + environment = BrowserEnvironment( config=BrowserConfig( initialization_url=f"{args.frontend_url.rstrip('/')}/initialize", extension_id=args.extension_id, ), + attach_to_existing=attach_to_existing, ) agent = Agent(environment=environment) run_started_ms = int(time.time() * 1000) try: - response = await agent.run(prompt=args.prompt, timeout=args.timeout) + try: + response = await agent.run(prompt=args.prompt, timeout=args.timeout) + except ValidationError as error: + _raise_clear_timestamp_error(error) run_finished_ms = int(time.time() * 1000) actions = _validate_trace(response.action_trace) @@ -158,7 +195,13 @@ async def _run(args: argparse.Namespace) -> None: f"client elapsed={run_finished_ms - run_started_ms} ms." ) finally: + owned_browser_pid = environment.browser_process_id await environment.close(timeout=30) + if owned_browser_pid is not None: + try: + os.kill(owned_browser_pid, signal.SIGTERM) + except ProcessLookupError: + pass def main() -> None: From b75b00ea0027031398f901766c9fe7e02f059842 Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Mon, 20 Jul 2026 18:13:07 -0700 Subject: [PATCH 06/15] Launch local extension for timing validation --- .../validate_operator_action_trace_timing.py | 222 ++++++++++++++++-- 1 file changed, 201 insertions(+), 21 deletions(-) diff --git a/scripts/validate_operator_action_trace_timing.py b/scripts/validate_operator_action_trace_timing.py index c975cee..4bebef5 100644 --- a/scripts/validate_operator_action_trace_timing.py +++ b/scripts/validate_operator_action_trace_timing.py @@ -13,12 +13,14 @@ import argparse import asyncio +import json import os -import signal import socket import time +import urllib.request from collections.abc import Sequence from datetime import UTC, datetime +from pathlib import Path from typing import NoReturn from narada import Agent, BrowserConfig, BrowserEnvironment @@ -29,7 +31,20 @@ DEFAULT_LOCAL_API_BASE_URL = "http://127.0.0.1:8000/fast/v2" DEFAULT_LOCAL_FRONTEND_URL = "http://localhost:3000" DEFAULT_DEV_EXTENSION_ID = "ijdopnjleolkjakldkjplfhniiohnccf" -DEFAULT_CDP_PORT = 9222 +DEFAULT_CDP_PORT = 9223 +WORKSPACE_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_LOCAL_EXTENSION_PATH = ( + WORKSPACE_ROOT + / "caddie" + / "src" + / "google" + / "chrome-extension" + / ".output" + / "chrome-mv3-dev" +) +DEFAULT_LOCAL_USER_DATA_DIR = ( + Path.home() / ".config" / "narada" / "user-data-dirs" / "action-trace-local-cft" +) DEFAULT_PROMPT = ( "Go to https://example.com, read the page heading, and tell me the heading." ) @@ -54,6 +69,32 @@ def _parse_args() -> argparse.Namespace: default=DEFAULT_DEV_EXTENSION_ID, help=f"Installed local extension ID (default: {DEFAULT_DEV_EXTENSION_ID}).", ) + parser.add_argument( + "--extension-path", + type=Path, + default=DEFAULT_LOCAL_EXTENSION_PATH, + help=f"Built local extension directory (default: {DEFAULT_LOCAL_EXTENSION_PATH}).", + ) + parser.add_argument( + "--browser-executable", + type=Path, + help=( + "Chromium or Chrome for Testing executable. By default, use the newest " + "Playwright Chromium installation." + ), + ) + parser.add_argument( + "--user-data-dir", + type=Path, + default=DEFAULT_LOCAL_USER_DATA_DIR, + help=f"Chrome profile for this test (default: {DEFAULT_LOCAL_USER_DATA_DIR}).", + ) + parser.add_argument( + "--cdp-port", + type=int, + default=DEFAULT_CDP_PORT, + help=f"Dedicated Chrome CDP port (default: {DEFAULT_CDP_PORT}).", + ) parser.add_argument( "--prompt", default=DEFAULT_PROMPT, @@ -73,6 +114,8 @@ def _parse_args() -> argparse.Namespace: parser.error("--frontend-url must not be empty.") if not args.extension_id: parser.error("--extension-id must not be empty.") + if args.cdp_port <= 0 or args.cdp_port > 65535: + parser.error("--cdp-port must be between 1 and 65535.") if "NARADA_API_KEY" not in os.environ: parser.error("Set NARADA_API_KEY before running this script.") if args.timeout <= 0: @@ -86,14 +129,136 @@ def _format_timestamp(timestamp_ms: int) -> str: return timestamp.isoformat(timespec="milliseconds") -def _is_cdp_browser_running() -> bool: +def _is_cdp_browser_running(cdp_port: int) -> bool: try: - with socket.create_connection(("127.0.0.1", DEFAULT_CDP_PORT), timeout=0.25): + with socket.create_connection(("127.0.0.1", cdp_port), timeout=0.25): return True except OSError: return False +def _find_extension_capable_browser(requested_path: Path | None) -> Path: + if requested_path is not None: + executable = requested_path.expanduser().resolve() + if not executable.is_file() or not os.access(executable, os.X_OK): + raise RuntimeError(f"Browser executable is not executable: {executable}") + return executable + + cache_roots = ( + Path.home() / "Library" / "Caches" / "ms-playwright", + Path.home() / ".cache" / "ms-playwright", + ) + executable_patterns = ( + "chromium-*/chrome-mac*/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing", + "chromium-*/chrome-mac*/Chromium.app/Contents/MacOS/Chromium", + "chromium-*/chrome-linux*/chrome", + "chromium-*/chrome-win*/chrome.exe", + ) + candidates = [ + executable + for cache_root in cache_roots + for pattern in executable_patterns + for executable in cache_root.glob(pattern) + if executable.is_file() and os.access(executable, os.X_OK) + ] + if not candidates: + raise RuntimeError( + "No extension-capable Playwright Chromium installation was found. " + "Run `uv run playwright install chromium` or pass --browser-executable." + ) + return max(candidates, key=lambda path: path.stat().st_mtime) + + +def _get_cdp_extension_ids(cdp_port: int) -> set[str]: + with urllib.request.urlopen( + f"http://127.0.0.1:{cdp_port}/json/list", + timeout=1, + ) as response: + targets = json.load(response) + + extension_prefix = "chrome-extension://" + return { + url.removeprefix(extension_prefix).partition("/")[0] + for target in targets + if isinstance(target, dict) + and isinstance((url := target.get("url")), str) + and url.startswith(extension_prefix) + } + + +async def _wait_for_extension(cdp_port: int, extension_id: str) -> None: + seen_extension_ids: set[str] = set() + for _ in range(50): + try: + seen_extension_ids = await asyncio.to_thread( + _get_cdp_extension_ids, + cdp_port, + ) + except OSError: + pass + if extension_id in seen_extension_ids: + return + await asyncio.sleep(0.1) + + seen = ", ".join(sorted(seen_extension_ids)) or "none" + raise RuntimeError( + f"Chrome on CDP port {cdp_port} did not load local extension " + f"{extension_id}. Extension IDs seen: {seen}." + ) + + +async def _launch_local_extension_browser( + *, + config: BrowserConfig, + extension_path: Path, +) -> asyncio.subprocess.Process: + extension_path = extension_path.resolve() + manifest_path = extension_path / "manifest.json" + if not manifest_path.is_file(): + raise RuntimeError( + f"Local extension build not found at {manifest_path}. " + "Start the extension dev server before running this script." + ) + + config.user_data_dir = str(Path(config.user_data_dir).expanduser().resolve()) + process = await asyncio.create_subprocess_exec( + config.executable_path, + f"--user-data-dir={config.user_data_dir}", + f"--profile-directory={config.profile_directory}", + f"--remote-debugging-port={config.cdp_port}", + f"--disable-extensions-except={extension_path}", + f"--load-extension={extension_path}", + "--no-default-browser-check", + "--no-first-run", + "about:blank", + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + start_new_session=True, + ) + + for _ in range(100): + if _is_cdp_browser_running(config.cdp_port): + try: + await _wait_for_extension(config.cdp_port, config.extension_id) + except Exception: + process.terminate() + await process.wait() + raise + return process + if process.returncode is not None: + raise RuntimeError( + f"Local Chrome exited during startup with code {process.returncode}." + ) + await asyncio.sleep(0.1) + + process.terminate() + await process.wait() + raise RuntimeError( + f"Timed out waiting for local Chrome on CDP port {config.cdp_port}." + ) + + def _raise_clear_timestamp_error(error: ValidationError) -> NoReturn: missing_timestamps = any( item["type"] == "missing" @@ -150,17 +315,29 @@ def _validate_trace( async def _run(args: argparse.Namespace) -> None: os.environ["NARADA_API_BASE_URL"] = args.base_url.rstrip("/") - attach_to_existing = _is_cdp_browser_running() - if attach_to_existing: - print(f"Reusing the local Chrome process on CDP port {DEFAULT_CDP_PORT}.") - - environment = BrowserEnvironment( - config=BrowserConfig( - initialization_url=f"{args.frontend_url.rstrip('/')}/initialize", - extension_id=args.extension_id, - ), - attach_to_existing=attach_to_existing, + browser_executable = _find_extension_capable_browser(args.browser_executable) + config = BrowserConfig( + executable_path=str(browser_executable), + user_data_dir=str(args.user_data_dir), + cdp_port=args.cdp_port, + initialization_url=f"{args.frontend_url.rstrip('/')}/initialize", + extension_id=args.extension_id, ) + browser_process: asyncio.subprocess.Process | None = None + if _is_cdp_browser_running(config.cdp_port): + await _wait_for_extension(config.cdp_port, config.extension_id) + print(f"Reusing the local dev Chrome process on CDP port {config.cdp_port}.") + else: + print( + f"Launching the local extension from {args.extension_path} " + f"in {browser_executable} on CDP port {config.cdp_port}." + ) + browser_process = await _launch_local_extension_browser( + config=config, + extension_path=args.extension_path, + ) + + environment = BrowserEnvironment(config=config, attach_to_existing=True) agent = Agent(environment=environment) run_started_ms = int(time.time() * 1000) @@ -195,13 +372,16 @@ async def _run(args: argparse.Namespace) -> None: f"client elapsed={run_finished_ms - run_started_ms} ms." ) finally: - owned_browser_pid = environment.browser_process_id - await environment.close(timeout=30) - if owned_browser_pid is not None: - try: - os.kill(owned_browser_pid, signal.SIGTERM) - except ProcessLookupError: - pass + try: + await environment.close(timeout=30) + finally: + if browser_process is not None and browser_process.returncode is None: + browser_process.terminate() + try: + await asyncio.wait_for(browser_process.wait(), timeout=10) + except TimeoutError: + browser_process.kill() + await browser_process.wait() def main() -> None: From 42c67d412be4c8f4c5501bfe437800fa29a12940 Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Mon, 20 Jul 2026 18:16:09 -0700 Subject: [PATCH 07/15] Remove timing validation script --- .../validate_operator_action_trace_timing.py | 393 ------------------ 1 file changed, 393 deletions(-) delete mode 100644 scripts/validate_operator_action_trace_timing.py diff --git a/scripts/validate_operator_action_trace_timing.py b/scripts/validate_operator_action_trace_timing.py deleted file mode 100644 index 4bebef5..0000000 --- a/scripts/validate_operator_action_trace_timing.py +++ /dev/null @@ -1,393 +0,0 @@ -"""Run Operator through the local frontend and validate action-trace timestamps. - -Example: - - export NARADA_API_KEY="..." - uv run python scripts/validate_operator_action_trace_timing.py - -The script uses the locally running caddie API, frontend, and development -extension. It launches a local Chrome window and closes that window on exit. -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import os -import socket -import time -import urllib.request -from collections.abc import Sequence -from datetime import UTC, datetime -from pathlib import Path -from typing import NoReturn - -from narada import Agent, BrowserConfig, BrowserEnvironment -from narada_core.tracing.model import OperatorActionTraceItem -from pydantic import ValidationError - -PRODUCTION_API_BASE_URL = "https://api.narada.ai/fast/v2" -DEFAULT_LOCAL_API_BASE_URL = "http://127.0.0.1:8000/fast/v2" -DEFAULT_LOCAL_FRONTEND_URL = "http://localhost:3000" -DEFAULT_DEV_EXTENSION_ID = "ijdopnjleolkjakldkjplfhniiohnccf" -DEFAULT_CDP_PORT = 9223 -WORKSPACE_ROOT = Path(__file__).resolve().parents[2] -DEFAULT_LOCAL_EXTENSION_PATH = ( - WORKSPACE_ROOT - / "caddie" - / "src" - / "google" - / "chrome-extension" - / ".output" - / "chrome-mv3-dev" -) -DEFAULT_LOCAL_USER_DATA_DIR = ( - Path.home() / ".config" / "narada" / "user-data-dirs" / "action-trace-local-cft" -) -DEFAULT_PROMPT = ( - "Go to https://example.com, read the page heading, and tell me the heading." -) - - -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Validate Operator action-trace timestamps through the local stack.", - ) - parser.add_argument( - "--base-url", - default=os.getenv("NARADA_API_BASE_URL", DEFAULT_LOCAL_API_BASE_URL), - help=(f"Local caddie API base URL (default: {DEFAULT_LOCAL_API_BASE_URL})."), - ) - parser.add_argument( - "--frontend-url", - default=DEFAULT_LOCAL_FRONTEND_URL, - help=f"Local frontend origin (default: {DEFAULT_LOCAL_FRONTEND_URL}).", - ) - parser.add_argument( - "--extension-id", - default=DEFAULT_DEV_EXTENSION_ID, - help=f"Installed local extension ID (default: {DEFAULT_DEV_EXTENSION_ID}).", - ) - parser.add_argument( - "--extension-path", - type=Path, - default=DEFAULT_LOCAL_EXTENSION_PATH, - help=f"Built local extension directory (default: {DEFAULT_LOCAL_EXTENSION_PATH}).", - ) - parser.add_argument( - "--browser-executable", - type=Path, - help=( - "Chromium or Chrome for Testing executable. By default, use the newest " - "Playwright Chromium installation." - ), - ) - parser.add_argument( - "--user-data-dir", - type=Path, - default=DEFAULT_LOCAL_USER_DATA_DIR, - help=f"Chrome profile for this test (default: {DEFAULT_LOCAL_USER_DATA_DIR}).", - ) - parser.add_argument( - "--cdp-port", - type=int, - default=DEFAULT_CDP_PORT, - help=f"Dedicated Chrome CDP port (default: {DEFAULT_CDP_PORT}).", - ) - parser.add_argument( - "--prompt", - default=DEFAULT_PROMPT, - help="Operator task to run. It should require at least one browser action.", - ) - parser.add_argument( - "--timeout", - type=int, - default=600, - help="Agent timeout in seconds (default: 600).", - ) - args = parser.parse_args() - - if args.base_url.rstrip("/") == PRODUCTION_API_BASE_URL: - parser.error("This local test refuses to run against the production API.") - if not args.frontend_url: - parser.error("--frontend-url must not be empty.") - if not args.extension_id: - parser.error("--extension-id must not be empty.") - if args.cdp_port <= 0 or args.cdp_port > 65535: - parser.error("--cdp-port must be between 1 and 65535.") - if "NARADA_API_KEY" not in os.environ: - parser.error("Set NARADA_API_KEY before running this script.") - if args.timeout <= 0: - parser.error("--timeout must be greater than zero.") - - return args - - -def _format_timestamp(timestamp_ms: int) -> str: - timestamp = datetime.fromtimestamp(timestamp_ms / 1000, tz=UTC) - return timestamp.isoformat(timespec="milliseconds") - - -def _is_cdp_browser_running(cdp_port: int) -> bool: - try: - with socket.create_connection(("127.0.0.1", cdp_port), timeout=0.25): - return True - except OSError: - return False - - -def _find_extension_capable_browser(requested_path: Path | None) -> Path: - if requested_path is not None: - executable = requested_path.expanduser().resolve() - if not executable.is_file() or not os.access(executable, os.X_OK): - raise RuntimeError(f"Browser executable is not executable: {executable}") - return executable - - cache_roots = ( - Path.home() / "Library" / "Caches" / "ms-playwright", - Path.home() / ".cache" / "ms-playwright", - ) - executable_patterns = ( - "chromium-*/chrome-mac*/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing", - "chromium-*/chrome-mac*/Chromium.app/Contents/MacOS/Chromium", - "chromium-*/chrome-linux*/chrome", - "chromium-*/chrome-win*/chrome.exe", - ) - candidates = [ - executable - for cache_root in cache_roots - for pattern in executable_patterns - for executable in cache_root.glob(pattern) - if executable.is_file() and os.access(executable, os.X_OK) - ] - if not candidates: - raise RuntimeError( - "No extension-capable Playwright Chromium installation was found. " - "Run `uv run playwright install chromium` or pass --browser-executable." - ) - return max(candidates, key=lambda path: path.stat().st_mtime) - - -def _get_cdp_extension_ids(cdp_port: int) -> set[str]: - with urllib.request.urlopen( - f"http://127.0.0.1:{cdp_port}/json/list", - timeout=1, - ) as response: - targets = json.load(response) - - extension_prefix = "chrome-extension://" - return { - url.removeprefix(extension_prefix).partition("/")[0] - for target in targets - if isinstance(target, dict) - and isinstance((url := target.get("url")), str) - and url.startswith(extension_prefix) - } - - -async def _wait_for_extension(cdp_port: int, extension_id: str) -> None: - seen_extension_ids: set[str] = set() - for _ in range(50): - try: - seen_extension_ids = await asyncio.to_thread( - _get_cdp_extension_ids, - cdp_port, - ) - except OSError: - pass - if extension_id in seen_extension_ids: - return - await asyncio.sleep(0.1) - - seen = ", ".join(sorted(seen_extension_ids)) or "none" - raise RuntimeError( - f"Chrome on CDP port {cdp_port} did not load local extension " - f"{extension_id}. Extension IDs seen: {seen}." - ) - - -async def _launch_local_extension_browser( - *, - config: BrowserConfig, - extension_path: Path, -) -> asyncio.subprocess.Process: - extension_path = extension_path.resolve() - manifest_path = extension_path / "manifest.json" - if not manifest_path.is_file(): - raise RuntimeError( - f"Local extension build not found at {manifest_path}. " - "Start the extension dev server before running this script." - ) - - config.user_data_dir = str(Path(config.user_data_dir).expanduser().resolve()) - process = await asyncio.create_subprocess_exec( - config.executable_path, - f"--user-data-dir={config.user_data_dir}", - f"--profile-directory={config.profile_directory}", - f"--remote-debugging-port={config.cdp_port}", - f"--disable-extensions-except={extension_path}", - f"--load-extension={extension_path}", - "--no-default-browser-check", - "--no-first-run", - "about:blank", - stdin=asyncio.subprocess.DEVNULL, - stdout=asyncio.subprocess.DEVNULL, - stderr=asyncio.subprocess.DEVNULL, - start_new_session=True, - ) - - for _ in range(100): - if _is_cdp_browser_running(config.cdp_port): - try: - await _wait_for_extension(config.cdp_port, config.extension_id) - except Exception: - process.terminate() - await process.wait() - raise - return process - if process.returncode is not None: - raise RuntimeError( - f"Local Chrome exited during startup with code {process.returncode}." - ) - await asyncio.sleep(0.1) - - process.terminate() - await process.wait() - raise RuntimeError( - f"Timed out waiting for local Chrome on CDP port {config.cdp_port}." - ) - - -def _raise_clear_timestamp_error(error: ValidationError) -> NoReturn: - missing_timestamps = any( - item["type"] == "missing" - and item["loc"] - and item["loc"][-1] in {"startTs", "endTs"} - for item in error.errors() - ) - if missing_timestamps: - raise RuntimeError( - "The local caddie process returned an action trace without timestamps. " - "Restart the backend after switching to " - "sp/operator-action-trace-timestamps, then run this script again." - ) from error - raise error - - -def _validate_trace( - trace: Sequence[object] | None, -) -> list[OperatorActionTraceItem]: - if not trace: - raise AssertionError("Operator returned no action trace.") - - actions: list[OperatorActionTraceItem] = [] - for index, item in enumerate(trace): - if not isinstance(item, OperatorActionTraceItem): - raise AssertionError( - f"Action {index} has unexpected trace type {type(item).__name__}." - ) - actions.append(item) - - if len(actions) < 2: - raise AssertionError( - "Expected at least one browser action followed by the done action." - ) - - for index, action in enumerate(actions): - if action.end_ts < action.start_ts: - raise AssertionError( - f"Action {index} ends before it starts: " - f"{action.start_ts} > {action.end_ts}." - ) - if index > 0 and action.start_ts != actions[index - 1].end_ts: - raise AssertionError( - f"Action {index} is not contiguous: startTs={action.start_ts}, " - f"previous endTs={actions[index - 1].end_ts}." - ) - - if not actions[-1].action.startswith("Done:"): - raise AssertionError("The final trace item is not the done action.") - - return actions - - -async def _run(args: argparse.Namespace) -> None: - os.environ["NARADA_API_BASE_URL"] = args.base_url.rstrip("/") - - browser_executable = _find_extension_capable_browser(args.browser_executable) - config = BrowserConfig( - executable_path=str(browser_executable), - user_data_dir=str(args.user_data_dir), - cdp_port=args.cdp_port, - initialization_url=f"{args.frontend_url.rstrip('/')}/initialize", - extension_id=args.extension_id, - ) - browser_process: asyncio.subprocess.Process | None = None - if _is_cdp_browser_running(config.cdp_port): - await _wait_for_extension(config.cdp_port, config.extension_id) - print(f"Reusing the local dev Chrome process on CDP port {config.cdp_port}.") - else: - print( - f"Launching the local extension from {args.extension_path} " - f"in {browser_executable} on CDP port {config.cdp_port}." - ) - browser_process = await _launch_local_extension_browser( - config=config, - extension_path=args.extension_path, - ) - - environment = BrowserEnvironment(config=config, attach_to_existing=True) - agent = Agent(environment=environment) - run_started_ms = int(time.time() * 1000) - - try: - try: - response = await agent.run(prompt=args.prompt, timeout=args.timeout) - except ValidationError as error: - _raise_clear_timestamp_error(error) - run_finished_ms = int(time.time() * 1000) - actions = _validate_trace(response.action_trace) - - print(f"Request: {response.request_id}") - print(f"Response: {response.text}") - print() - print("Operator action timing:") - for index, action in enumerate(actions, start=1): - duration_ms = action.end_ts - action.start_ts - print( - f"{index:>2}. {action.action}\n" - f" start: {_format_timestamp(action.start_ts)} " - f"({action.start_ts})\n" - f" end: {_format_timestamp(action.end_ts)} " - f"({action.end_ts})\n" - f" duration: {duration_ms} ms" - ) - - total_duration_ms = actions[-1].end_ts - actions[0].start_ts - print() - print( - f"PASS: {len(actions)} contiguous actions; " - f"trace duration={total_duration_ms} ms; " - f"client elapsed={run_finished_ms - run_started_ms} ms." - ) - finally: - try: - await environment.close(timeout=30) - finally: - if browser_process is not None and browser_process.returncode is None: - browser_process.terminate() - try: - await asyncio.wait_for(browser_process.wait(), timeout=10) - except TimeoutError: - browser_process.kill() - await browser_process.wait() - - -def main() -> None: - args = _parse_args() - asyncio.run(_run(args)) - - -if __name__ == "__main__": - main() From a6a427e64f8a6347fadbd7f98024b7c7c16f75dc Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Mon, 20 Jul 2026 19:01:36 -0700 Subject: [PATCH 08/15] Expose action trace timestamps as ISO strings --- .../narada-core/src/narada_core/models.py | 4 +- .../src/narada_core/tracing/model.py | 19 +++++++-- .../tests/test_cloud_browser.py | 4 +- packages/narada/tests/test_action_trace.py | 39 +++++++++++++++---- packages/narada/tests/test_cloud_browser.py | 4 +- 5 files changed, 53 insertions(+), 17 deletions(-) diff --git a/packages/narada-core/src/narada_core/models.py b/packages/narada-core/src/narada_core/models.py index 38ef3c0..d95729c 100644 --- a/packages/narada-core/src/narada_core/models.py +++ b/packages/narada-core/src/narada_core/models.py @@ -96,8 +96,8 @@ class RemoteDispatchChatHistoryItem(TypedDict): class OperatorActionTraceItem(TypedDict): url: str action: str - startTs: int - endTs: int + startTs: str + endTs: str class GoToUrlTrace(TypedDict): diff --git a/packages/narada-core/src/narada_core/tracing/model.py b/packages/narada-core/src/narada_core/tracing/model.py index b52e6e1..2fa4e99 100644 --- a/packages/narada-core/src/narada_core/tracing/model.py +++ b/packages/narada-core/src/narada_core/tracing/model.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime, timedelta from typing import Annotated, Any, Literal from pydantic import ( @@ -29,12 +30,14 @@ def _normalize_agent_type(agent_type: object) -> str: class OperatorActionTraceItem(BaseModel): url: str action: str - start_ts: NonNegativeInt = Field(alias="startTs") - end_ts: NonNegativeInt = Field(alias="endTs") + start_ts: str = Field(alias="startTs") + end_ts: str = Field(alias="endTs") @model_validator(mode="after") def _check_timestamp_range(self) -> OperatorActionTraceItem: - if self.end_ts < self.start_ts: + start_timestamp = _parse_utc_iso_timestamp(self.start_ts, field_name="startTs") + end_timestamp = _parse_utc_iso_timestamp(self.end_ts, field_name="endTs") + if end_timestamp < start_timestamp: raise ValueError( f"OperatorActionTraceItem: endTs ({self.end_ts}) must be >= " f"startTs ({self.start_ts})" @@ -42,6 +45,16 @@ def _check_timestamp_range(self) -> OperatorActionTraceItem: return self +def _parse_utc_iso_timestamp(value: str, *, field_name: str) -> datetime: + try: + timestamp = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise ValueError(f"{field_name} must be an ISO 8601 timestamp") from error + if timestamp.utcoffset() != timedelta(0): + raise ValueError(f"{field_name} must be a UTC ISO 8601 timestamp") + return timestamp + + class GoToUrlTrace(BaseModel): step_type: Literal["goToUrl"] url: str diff --git a/packages/narada-pyodide/tests/test_cloud_browser.py b/packages/narada-pyodide/tests/test_cloud_browser.py index 438da1c..ac0bb4f 100644 --- a/packages/narada-pyodide/tests/test_cloud_browser.py +++ b/packages/narada-pyodide/tests/test_cloud_browser.py @@ -536,7 +536,7 @@ async def test_agent_run_emits_combined_critic_workflow_trace( "workflowName": "Main Workflow", "runtime": "gui", "status": "success", - "startTs": 100, + "startTs": "2026-01-01T00:00:00.000Z", "children": [], } critic_workflow_trace = { @@ -544,7 +544,7 @@ async def test_agent_run_emits_combined_critic_workflow_trace( "workflowName": "Critic Workflow", "runtime": "gui", "status": "success", - "startTs": 200, + "startTs": "2026-01-01T00:00:01.000Z", "children": [], } pyfetch = AsyncMock( diff --git a/packages/narada/tests/test_action_trace.py b/packages/narada/tests/test_action_trace.py index 7843ae2..c961690 100644 --- a/packages/narada/tests/test_action_trace.py +++ b/packages/narada/tests/test_action_trace.py @@ -11,28 +11,40 @@ def test_parse_operator_action_trace_exposes_timestamps() -> None: { "url": "https://example.com", "action": "Clicked Submit", - "startTs": 1_000, - "endTs": 2_500, + "startTs": "2026-07-20T17:00:00.000Z", + "endTs": "2026-07-20T17:00:01.500Z", } ] ) item = trace[0] assert isinstance(item, OperatorActionTraceItem) - assert item.start_ts == 1_000 - assert item.end_ts == 2_500 + assert item.start_ts == "2026-07-20T17:00:00.000Z" + assert item.end_ts == "2026-07-20T17:00:01.500Z" @pytest.mark.parametrize( "timestamps", [ - {"startTs": 1_000}, - {"endTs": 2_000}, - {"startTs": 2_000, "endTs": 1_000}, + {"startTs": "2026-07-20T17:00:00.000Z"}, + {"endTs": "2026-07-20T17:00:01.000Z"}, + { + "startTs": "2026-07-20T17:00:02.000Z", + "endTs": "2026-07-20T17:00:01.000Z", + }, + { + "startTs": "not-a-timestamp", + "endTs": "2026-07-20T17:00:01.000Z", + }, + { + "startTs": "2026-07-20T17:00:00", + "endTs": "2026-07-20T17:00:01", + }, + {"startTs": 1_000, "endTs": 2_000}, ], ) def test_operator_action_trace_rejects_invalid_timestamp_ranges( - timestamps: dict[str, int], + timestamps: dict[str, str | int], ) -> None: with pytest.raises(ValidationError): OperatorActionTraceItem( @@ -40,3 +52,14 @@ def test_operator_action_trace_rejects_invalid_timestamp_ranges( action="Clicked Submit", **timestamps, ) + + +def test_operator_action_trace_accepts_explicit_utc_offset() -> None: + item = OperatorActionTraceItem( + url="https://example.com", + action="Clicked Submit", + startTs="2026-07-20T17:00:00.000+00:00", + endTs="2026-07-20T17:00:01.000+00:00", + ) + + assert item.start_ts.endswith("+00:00") diff --git a/packages/narada/tests/test_cloud_browser.py b/packages/narada/tests/test_cloud_browser.py index 7c4c908..d76707b 100644 --- a/packages/narada/tests/test_cloud_browser.py +++ b/packages/narada/tests/test_cloud_browser.py @@ -844,7 +844,7 @@ async def test_agent_run_appends_critic_workflow_trace( "workflowName": "Main Workflow", "runtime": "gui", "status": "success", - "startTs": 100, + "startTs": "2026-01-01T00:00:00.000Z", "children": [], } critic_workflow_trace = { @@ -852,7 +852,7 @@ async def test_agent_run_appends_critic_workflow_trace( "workflowName": "Critic Workflow", "runtime": "gui", "status": "success", - "startTs": 200, + "startTs": "2026-01-01T00:00:01.000Z", "children": [], } env = CloudBrowserEnvironment( From 4a51fa32b8256c037888746adf3692f6accbf7b5 Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Mon, 20 Jul 2026 19:43:02 -0700 Subject: [PATCH 09/15] Expose action trace duration --- .../narada-core/src/narada_core/models.py | 1 + .../src/narada_core/tracing/model.py | 9 ++++++++ packages/narada/tests/test_action_trace.py | 23 +++++++++++++++---- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/packages/narada-core/src/narada_core/models.py b/packages/narada-core/src/narada_core/models.py index d95729c..cc5d7e9 100644 --- a/packages/narada-core/src/narada_core/models.py +++ b/packages/narada-core/src/narada_core/models.py @@ -98,6 +98,7 @@ class OperatorActionTraceItem(TypedDict): action: str startTs: str endTs: str + durationMs: int class GoToUrlTrace(TypedDict): diff --git a/packages/narada-core/src/narada_core/tracing/model.py b/packages/narada-core/src/narada_core/tracing/model.py index 2fa4e99..e22890e 100644 --- a/packages/narada-core/src/narada_core/tracing/model.py +++ b/packages/narada-core/src/narada_core/tracing/model.py @@ -32,6 +32,7 @@ class OperatorActionTraceItem(BaseModel): action: str start_ts: str = Field(alias="startTs") end_ts: str = Field(alias="endTs") + duration_ms: NonNegativeInt = Field(alias="durationMs") @model_validator(mode="after") def _check_timestamp_range(self) -> OperatorActionTraceItem: @@ -42,6 +43,14 @@ def _check_timestamp_range(self) -> OperatorActionTraceItem: f"OperatorActionTraceItem: endTs ({self.end_ts}) must be >= " f"startTs ({self.start_ts})" ) + expected_duration_ms = round( + (end_timestamp - start_timestamp).total_seconds() * 1000 + ) + if self.duration_ms != expected_duration_ms: + raise ValueError( + f"OperatorActionTraceItem: durationMs ({self.duration_ms}) must equal " + f"endTs - startTs ({expected_duration_ms})" + ) return self diff --git a/packages/narada/tests/test_action_trace.py b/packages/narada/tests/test_action_trace.py index c961690..22f2f10 100644 --- a/packages/narada/tests/test_action_trace.py +++ b/packages/narada/tests/test_action_trace.py @@ -13,6 +13,7 @@ def test_parse_operator_action_trace_exposes_timestamps() -> None: "action": "Clicked Submit", "startTs": "2026-07-20T17:00:00.000Z", "endTs": "2026-07-20T17:00:01.500Z", + "durationMs": 1_500, } ] ) @@ -21,29 +22,42 @@ def test_parse_operator_action_trace_exposes_timestamps() -> None: assert isinstance(item, OperatorActionTraceItem) assert item.start_ts == "2026-07-20T17:00:00.000Z" assert item.end_ts == "2026-07-20T17:00:01.500Z" + assert item.duration_ms == 1_500 @pytest.mark.parametrize( "timestamps", [ - {"startTs": "2026-07-20T17:00:00.000Z"}, - {"endTs": "2026-07-20T17:00:01.000Z"}, + {"startTs": "2026-07-20T17:00:00.000Z", "durationMs": 1_000}, + {"endTs": "2026-07-20T17:00:01.000Z", "durationMs": 1_000}, { "startTs": "2026-07-20T17:00:02.000Z", "endTs": "2026-07-20T17:00:01.000Z", + "durationMs": 0, }, { "startTs": "not-a-timestamp", "endTs": "2026-07-20T17:00:01.000Z", + "durationMs": 1_000, }, { "startTs": "2026-07-20T17:00:00", "endTs": "2026-07-20T17:00:01", + "durationMs": 1_000, + }, + {"startTs": 1_000, "endTs": 2_000, "durationMs": 1_000}, + { + "startTs": "2026-07-20T17:00:00.000Z", + "endTs": "2026-07-20T17:00:01.000Z", + }, + { + "startTs": "2026-07-20T17:00:00.000Z", + "endTs": "2026-07-20T17:00:01.000Z", + "durationMs": 999, }, - {"startTs": 1_000, "endTs": 2_000}, ], ) -def test_operator_action_trace_rejects_invalid_timestamp_ranges( +def test_operator_action_trace_rejects_invalid_timing( timestamps: dict[str, str | int], ) -> None: with pytest.raises(ValidationError): @@ -60,6 +74,7 @@ def test_operator_action_trace_accepts_explicit_utc_offset() -> None: action="Clicked Submit", startTs="2026-07-20T17:00:00.000+00:00", endTs="2026-07-20T17:00:01.000+00:00", + durationMs=1_000, ) assert item.start_ts.endswith("+00:00") From 34fa743d5b3a9ef5150fe33c7517e21851b36851 Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Tue, 21 Jul 2026 13:03:57 -0700 Subject: [PATCH 10/15] Describe Operator completion boundary semantics --- .../src/narada_core/tracing/model.py | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/narada-core/src/narada_core/tracing/model.py b/packages/narada-core/src/narada_core/tracing/model.py index e22890e..602aa9f 100644 --- a/packages/narada-core/src/narada_core/tracing/model.py +++ b/packages/narada-core/src/narada_core/tracing/model.py @@ -30,9 +30,24 @@ def _normalize_agent_type(agent_type: object) -> str: class OperatorActionTraceItem(BaseModel): url: str action: str - start_ts: str = Field(alias="startTs") - end_ts: str = Field(alias="endTs") - duration_ms: NonNegativeInt = Field(alias="durationMs") + start_ts: str = Field( + alias="startTs", + description=( + "UTC boundary inherited from the prior Operator turn completion, or " + "from the first generate request receipt for the first turn." + ), + ) + end_ts: str = Field( + alias="endTs", + description=( + "UTC completion boundary: backend coroutine return or receipt of the " + "next frontend acknowledgement." + ), + ) + duration_ms: NonNegativeInt = Field( + alias="durationMs", + description="Elapsed milliseconds between startTs and endTs.", + ) @model_validator(mode="after") def _check_timestamp_range(self) -> OperatorActionTraceItem: From b9a8bde437fb0616e1ac4ab7e69e4b8e3181d46e Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Tue, 21 Jul 2026 13:24:03 -0700 Subject: [PATCH 11/15] Revert "Describe Operator completion boundary semantics" This reverts commit 34fa743d5b3a9ef5150fe33c7517e21851b36851. --- .../src/narada_core/tracing/model.py | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/packages/narada-core/src/narada_core/tracing/model.py b/packages/narada-core/src/narada_core/tracing/model.py index 602aa9f..e22890e 100644 --- a/packages/narada-core/src/narada_core/tracing/model.py +++ b/packages/narada-core/src/narada_core/tracing/model.py @@ -30,24 +30,9 @@ def _normalize_agent_type(agent_type: object) -> str: class OperatorActionTraceItem(BaseModel): url: str action: str - start_ts: str = Field( - alias="startTs", - description=( - "UTC boundary inherited from the prior Operator turn completion, or " - "from the first generate request receipt for the first turn." - ), - ) - end_ts: str = Field( - alias="endTs", - description=( - "UTC completion boundary: backend coroutine return or receipt of the " - "next frontend acknowledgement." - ), - ) - duration_ms: NonNegativeInt = Field( - alias="durationMs", - description="Elapsed milliseconds between startTs and endTs.", - ) + start_ts: str = Field(alias="startTs") + end_ts: str = Field(alias="endTs") + duration_ms: NonNegativeInt = Field(alias="durationMs") @model_validator(mode="after") def _check_timestamp_range(self) -> OperatorActionTraceItem: From 8cad24feccab2319f61a56089907567a2d672581 Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Tue, 21 Jul 2026 15:58:26 -0700 Subject: [PATCH 12/15] Remove unrelated workflow timing fixtures --- packages/narada-pyodide/tests/test_cloud_browser.py | 4 ++-- packages/narada/tests/test_cloud_browser.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/narada-pyodide/tests/test_cloud_browser.py b/packages/narada-pyodide/tests/test_cloud_browser.py index ac0bb4f..438da1c 100644 --- a/packages/narada-pyodide/tests/test_cloud_browser.py +++ b/packages/narada-pyodide/tests/test_cloud_browser.py @@ -536,7 +536,7 @@ async def test_agent_run_emits_combined_critic_workflow_trace( "workflowName": "Main Workflow", "runtime": "gui", "status": "success", - "startTs": "2026-01-01T00:00:00.000Z", + "startTs": 100, "children": [], } critic_workflow_trace = { @@ -544,7 +544,7 @@ async def test_agent_run_emits_combined_critic_workflow_trace( "workflowName": "Critic Workflow", "runtime": "gui", "status": "success", - "startTs": "2026-01-01T00:00:01.000Z", + "startTs": 200, "children": [], } pyfetch = AsyncMock( diff --git a/packages/narada/tests/test_cloud_browser.py b/packages/narada/tests/test_cloud_browser.py index d76707b..7c4c908 100644 --- a/packages/narada/tests/test_cloud_browser.py +++ b/packages/narada/tests/test_cloud_browser.py @@ -844,7 +844,7 @@ async def test_agent_run_appends_critic_workflow_trace( "workflowName": "Main Workflow", "runtime": "gui", "status": "success", - "startTs": "2026-01-01T00:00:00.000Z", + "startTs": 100, "children": [], } critic_workflow_trace = { @@ -852,7 +852,7 @@ async def test_agent_run_appends_critic_workflow_trace( "workflowName": "Critic Workflow", "runtime": "gui", "status": "success", - "startTs": "2026-01-01T00:00:01.000Z", + "startTs": 200, "children": [], } env = CloudBrowserEnvironment( From 506d189452eed6939f311b00c263633d058f1001 Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Tue, 21 Jul 2026 16:08:02 -0700 Subject: [PATCH 13/15] Restore workflow trace ISO fixtures --- packages/narada-pyodide/tests/test_cloud_browser.py | 4 ++-- packages/narada/tests/test_cloud_browser.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/narada-pyodide/tests/test_cloud_browser.py b/packages/narada-pyodide/tests/test_cloud_browser.py index 438da1c..ac0bb4f 100644 --- a/packages/narada-pyodide/tests/test_cloud_browser.py +++ b/packages/narada-pyodide/tests/test_cloud_browser.py @@ -536,7 +536,7 @@ async def test_agent_run_emits_combined_critic_workflow_trace( "workflowName": "Main Workflow", "runtime": "gui", "status": "success", - "startTs": 100, + "startTs": "2026-01-01T00:00:00.000Z", "children": [], } critic_workflow_trace = { @@ -544,7 +544,7 @@ async def test_agent_run_emits_combined_critic_workflow_trace( "workflowName": "Critic Workflow", "runtime": "gui", "status": "success", - "startTs": 200, + "startTs": "2026-01-01T00:00:01.000Z", "children": [], } pyfetch = AsyncMock( diff --git a/packages/narada/tests/test_cloud_browser.py b/packages/narada/tests/test_cloud_browser.py index 7c4c908..d76707b 100644 --- a/packages/narada/tests/test_cloud_browser.py +++ b/packages/narada/tests/test_cloud_browser.py @@ -844,7 +844,7 @@ async def test_agent_run_appends_critic_workflow_trace( "workflowName": "Main Workflow", "runtime": "gui", "status": "success", - "startTs": 100, + "startTs": "2026-01-01T00:00:00.000Z", "children": [], } critic_workflow_trace = { @@ -852,7 +852,7 @@ async def test_agent_run_appends_critic_workflow_trace( "workflowName": "Critic Workflow", "runtime": "gui", "status": "success", - "startTs": 200, + "startTs": "2026-01-01T00:00:01.000Z", "children": [], } env = CloudBrowserEnvironment( From 3090b5f43dbd4a0b03bc1dcf937dcf4211e6b7d1 Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Tue, 21 Jul 2026 16:27:27 -0700 Subject: [PATCH 14/15] Keep workflow trace timestamps unchanged --- packages/narada-pyodide/tests/test_cloud_browser.py | 4 ++-- packages/narada/tests/test_cloud_browser.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/narada-pyodide/tests/test_cloud_browser.py b/packages/narada-pyodide/tests/test_cloud_browser.py index ac0bb4f..438da1c 100644 --- a/packages/narada-pyodide/tests/test_cloud_browser.py +++ b/packages/narada-pyodide/tests/test_cloud_browser.py @@ -536,7 +536,7 @@ async def test_agent_run_emits_combined_critic_workflow_trace( "workflowName": "Main Workflow", "runtime": "gui", "status": "success", - "startTs": "2026-01-01T00:00:00.000Z", + "startTs": 100, "children": [], } critic_workflow_trace = { @@ -544,7 +544,7 @@ async def test_agent_run_emits_combined_critic_workflow_trace( "workflowName": "Critic Workflow", "runtime": "gui", "status": "success", - "startTs": "2026-01-01T00:00:01.000Z", + "startTs": 200, "children": [], } pyfetch = AsyncMock( diff --git a/packages/narada/tests/test_cloud_browser.py b/packages/narada/tests/test_cloud_browser.py index d76707b..7c4c908 100644 --- a/packages/narada/tests/test_cloud_browser.py +++ b/packages/narada/tests/test_cloud_browser.py @@ -844,7 +844,7 @@ async def test_agent_run_appends_critic_workflow_trace( "workflowName": "Main Workflow", "runtime": "gui", "status": "success", - "startTs": "2026-01-01T00:00:00.000Z", + "startTs": 100, "children": [], } critic_workflow_trace = { @@ -852,7 +852,7 @@ async def test_agent_run_appends_critic_workflow_trace( "workflowName": "Critic Workflow", "runtime": "gui", "status": "success", - "startTs": "2026-01-01T00:00:01.000Z", + "startTs": 200, "children": [], } env = CloudBrowserEnvironment( From 2e86688e926f1651c7edb4c778a90fd5a3e045ad Mon Sep 17 00:00:00 2001 From: soon-dubu Date: Tue, 21 Jul 2026 19:03:29 -0700 Subject: [PATCH 15/15] Bump SDK packages for action timing --- packages/narada-core/pyproject.toml | 2 +- packages/narada-pyodide/pyproject.toml | 4 ++-- packages/narada/pyproject.toml | 4 ++-- uv.lock | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/narada-core/pyproject.toml b/packages/narada-core/pyproject.toml index e088258..b9ac9e6 100644 --- a/packages/narada-core/pyproject.toml +++ b/packages/narada-core/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "narada-core" -version = "0.1.4" +version = "0.1.5" description = "Code shared by the `narada` and `narada-pyodide` packages." license = "Apache-2.0" readme = "README.md" diff --git a/packages/narada-pyodide/pyproject.toml b/packages/narada-pyodide/pyproject.toml index 8c48183..a6e5c8a 100644 --- a/packages/narada-pyodide/pyproject.toml +++ b/packages/narada-pyodide/pyproject.toml @@ -1,14 +1,14 @@ [project] name = "narada-pyodide" -version = "0.1.4" +version = "0.1.5" description = "Pyodide-compatible Python client SDK for Narada" license = "Apache-2.0" readme = "README.md" authors = [{ name = "Narada", email = "support@narada.ai" }] requires-python = ">=3.12" dependencies = [ - "narada-core==0.1.4", + "narada-core==0.1.5", # Must be a supported version in https://pyodide.org/en/stable/usage/packages-in-pyodide.html "packaging==24.2", ] diff --git a/packages/narada/pyproject.toml b/packages/narada/pyproject.toml index 06dd4f6..6d915f8 100644 --- a/packages/narada/pyproject.toml +++ b/packages/narada/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "narada" -version = "0.2.7" +version = "0.2.8" description = "Python client SDK for Narada" license = "Apache-2.0" readme = "README.md" authors = [{ name = "Narada", email = "support@narada.ai" }] requires-python = ">=3.12" dependencies = [ - "narada-core==0.1.4", + "narada-core==0.1.5", "aiohttp>=3.12.13", "playwright>=1.53.0", "rich>=14.0.0", diff --git a/uv.lock b/uv.lock index 99e42c4..ad9ba35 100644 --- a/uv.lock +++ b/uv.lock @@ -312,7 +312,7 @@ wheels = [ [[package]] name = "narada" -version = "0.2.7" +version = "0.2.8" source = { editable = "packages/narada" } dependencies = [ { name = "aiohttp" }, @@ -345,7 +345,7 @@ dev = [ [[package]] name = "narada-core" -version = "0.1.4" +version = "0.1.5" source = { editable = "packages/narada-core" } dependencies = [ { name = "pydantic" }, @@ -356,7 +356,7 @@ requires-dist = [{ name = "pydantic", specifier = "==2.12.5" }] [[package]] name = "narada-pyodide" -version = "0.1.4" +version = "0.1.5" source = { editable = "packages/narada-pyodide" } dependencies = [ { name = "narada-core" },