feat: ref-based interaction tools (fill, hover, press_key, select, check, scroll)

e2e/mcp/browser-tools.test.ts

@@ -292,9 +292,11 @@ describe('MCP E2E: browser control tools', () => {
       expect(mcp.getText(result)).toMatch(/clicked at/i);
     });
 
-    it('returns an error when neither selector nor coordinates are provided', async () => {
+    it('returns an error when neither ref, selector, nor coordinates are provided', async () => {
       const result = await mcp.callTool('browser_click', {});
-      expect(mcp.getText(result)).toMatch(/selector or x\/y/i);
+      expect(mcp.getText(result)).toMatch(
+        /ref.*selector.*x\/y|selector.*x\/y/i
+      );
     });
   });
 });

src/browser/interaction/events.ts

@@ -0,0 +1,79 @@
+import type { Client } from 'chrome-remote-interface';
+
+/**
+ * Fire framework-compatible input events on an element.
+ * Uses the native value setter to work with React's synthetic event system,
+ * then dispatches native DOM events with bubbling enabled for Vue/Svelte.
+ */
+export async function fireInputEvents(
+  client: Client,
+  objectId: string
+): Promise<void> {
+  await client.Runtime.callFunctionOn({
+    objectId,
+    functionDeclaration: `function() {
+      this.dispatchEvent(new Event('input', { bubbles: true }));
+      this.dispatchEvent(new Event('change', { bubbles: true }));
+    }`,
+    awaitPromise: false
+  });
+}
+
+/**
+ * Set a form field value using the native setter for React compatibility,
+ * then fire input/change events.
+ */
+export async function setFieldValue(
+  client: Client,
+  objectId: string,
+  value: string
+): Promise<void> {
+  await client.Runtime.callFunctionOn({
+    objectId,
+    functionDeclaration: `function(newValue) {
+      var tag = this.tagName.toLowerCase();
+      if (tag === 'select') {
+        // Select element — set by value
+        for (var i = 0; i < this.options.length; i++) {
+          if (this.options[i].value === newValue) {
+            this.selectedIndex = i;
+            break;
+          }
+        }
+      } else if (this.getAttribute('contenteditable') !== null) {
+        this.textContent = newValue;
+      } else {
+        // Use native setter for React compatibility
+        var proto = tag === 'textarea'
+          ? HTMLTextAreaElement.prototype
+          : HTMLInputElement.prototype;
+        var setter = Object.getOwnPropertyDescriptor(proto, 'value');
+        if (setter && setter.set) {
+          setter.set.call(this, newValue);
+        } else {
+          this.value = newValue;
+        }
+      }
+      this.dispatchEvent(new Event('input', { bubbles: true }));
+      this.dispatchEvent(new Event('change', { bubbles: true }));
+    }`,
+    arguments: [{ value }],
+    awaitPromise: false
+  });
+}
+
+/**
+ * Get the tag name of an element by its remote object ID.
+ */
+export async function getTagName(
+  client: Client,
+  objectId: string
+): Promise<string> {
+  const { result } = await client.Runtime.callFunctionOn({
+    objectId,
+    functionDeclaration: `function() { return this.tagName ? this.tagName.toLowerCase() : ''; }`,
+    returnByValue: true,
+    awaitPromise: false
+  });
+  return String(result.value ?? '');
+}

src/browser/interaction/ref-resolver.ts

@@ -0,0 +1,53 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../inspect/element-ref.js';
+
+export interface ResolvedElement {
+  backendNodeId: number;
+  objectId: string;
+  x: number;
+  y: number;
+}
+
+/**
+ * Resolve an element ref to coordinates and a remote object ID.
+ * Scrolls the element into view first so click coordinates are valid.
+ */
+export async function resolveRef(
+  client: Client,
+  refMap: ElementRefMap,
+  ref: string
+): Promise<ResolvedElement> {
+  const backendNodeId = refMap.resolve(ref);
+
+  // Scroll into view
+  await client.DOM.scrollIntoViewIfNeeded({ backendNodeId });
+
+  // Get center coordinates from box model
+  const { model } = await client.DOM.getBoxModel({ backendNodeId });
+  const content = model.content;
+  const x = (content[0] + content[2] + content[4] + content[6]) / 4;
+  const y = (content[1] + content[3] + content[5] + content[7]) / 4;
+
+  // Get remote object for Runtime.callFunctionOn
+  const { object } = await client.DOM.resolveNode({ backendNodeId });
+  if (!object.objectId) {
+    throw new Error(`Could not resolve ref "${ref}" to a remote object`);
+  }
+
+  return {
+    backendNodeId,
+    objectId: object.objectId,
+    x: Math.round(x),
+    y: Math.round(y)
+  };
+}
+
+/**
+ * Focus an element by ref.
+ */
+export async function focusRef(
+  client: Client,
+  backendNodeId: number
+): Promise<void> {
+  await client.DOM.focus({ backendNodeId });
+}

src/commands/check.ts

@@ -0,0 +1,59 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { resolveRef } from '../browser/interaction/ref-resolver.js';
+
+export interface CheckParams {
+  ref: string;
+  checked?: boolean;
+}
+
+export interface CheckResult {
+  ref: string;
+  checked: boolean;
+}
+
+/**
+ * Toggle a checkbox or radio button by ref.
+ * If `checked` is specified, sets it to that state (no-op if already matching).
+ * If omitted, toggles the current state.
+ */
+export async function executeCheck(
+  client: Client,
+  refMap: ElementRefMap,
+  params: CheckParams
+): Promise<CheckResult> {
+  const resolved = await resolveRef(client, refMap, params.ref);
+
+  const { result, exceptionDetails } = await client.Runtime.callFunctionOn({
+    objectId: resolved.objectId,
+    functionDeclaration: `function(desiredState) {
+      var tag = this.tagName.toLowerCase();
+      var type = (this.type || '').toLowerCase();
+      if (tag !== 'input' || (type !== 'checkbox' && type !== 'radio')) {
+        throw new Error('Element is not a checkbox or radio button');
+      }
+      var current = this.checked;
+      var target = desiredState !== null ? desiredState : !current;
+      if (current !== target) {
+        this.click();
+      }
+      return this.checked;
+    }`,
+    arguments: [{ value: params.checked ?? null }],
+    returnByValue: true,
+    awaitPromise: false
+  });
+
+  if (exceptionDetails) {
+    throw new Error(
+      exceptionDetails.exception?.description ??
+        exceptionDetails.text ??
+        'Failed to toggle checkbox'
+    );
+  }
+
+  return {
+    ref: params.ref,
+    checked: result.value as boolean
+  };
+}

src/commands/click.ts

@@ -1,8 +1,11 @@
 import { z } from 'zod';
 import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { resolveRef } from '../browser/interaction/ref-resolver.js';
 
 export const ClickParamsSchema = z
   .object({
+    ref: z.string().optional(),
     selector: z.string().optional(),
     x: z.number().optional(),
     y: z.number().optional(),
@@ -11,9 +14,12 @@ export const ClickParamsSchema = z
     timeoutMs: z.number().int().positive().default(5000)
   })
   .refine(
-    (v) => v.selector !== undefined || (v.x !== undefined && v.y !== undefined),
+    (v) =>
+      v.ref !== undefined ||
+      v.selector !== undefined ||
+      (v.x !== undefined && v.y !== undefined),
     {
-      message: 'Either selector or x/y coordinates are required'
+      message: 'Either ref, selector, or x/y coordinates are required'
     }
   );
 
@@ -52,12 +58,20 @@ async function waitForSelector(
 
 export async function executeClick(
   client: Client,
-  params: ClickParams
+  params: ClickParams,
+  refMap?: ElementRefMap
 ): Promise<ClickResult> {
   let x: number;
   let y: number;
 
-  if (params.selector !== undefined) {
+  if (params.ref !== undefined) {
+    if (!refMap) {
+      throw new Error('Element ref map is required when using ref parameter');
+    }
+    const resolved = await resolveRef(client, refMap, params.ref);
+    x = resolved.x;
+    y = resolved.y;
+  } else if (params.selector !== undefined) {
     const pos = await waitForSelector(
       client,
       params.selector,

src/commands/fill.ts

@@ -0,0 +1,34 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { resolveRef, focusRef } from '../browser/interaction/ref-resolver.js';
+import { setFieldValue } from '../browser/interaction/events.js';
+
+export interface FillParams {
+  ref: string;
+  value: string;
+}
+
+export interface FillResult {
+  ref: string;
+  value: string;
+}
+
+/**
+ * Fill a form field by ref — clears the existing value and sets a new one.
+ * Fires framework-compatible input/change events.
+ */
+export async function executeFill(
+  client: Client,
+  refMap: ElementRefMap,
+  params: FillParams
+): Promise<FillResult> {
+  const resolved = await resolveRef(client, refMap, params.ref);
+
+  // Focus the element
+  await focusRef(client, resolved.backendNodeId);
+
+  // Set value and fire events
+  await setFieldValue(client, resolved.objectId, params.value);
+
+  return { ref: params.ref, value: params.value };
+}

src/commands/hover.ts

@@ -0,0 +1,42 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { resolveRef } from '../browser/interaction/ref-resolver.js';
+
+export interface HoverParams {
+  ref?: string;
+  x?: number;
+  y?: number;
+}
+
+export interface HoverResult {
+  x: number;
+  y: number;
+}
+
+export async function executeHover(
+  client: Client,
+  refMap: ElementRefMap,
+  params: HoverParams
+): Promise<HoverResult> {
+  let x: number;
+  let y: number;
+
+  if (params.ref) {
+    const resolved = await resolveRef(client, refMap, params.ref);
+    x = resolved.x;
+    y = resolved.y;
+  } else if (params.x !== undefined && params.y !== undefined) {
+    x = params.x;
+    y = params.y;
+  } else {
+    throw new Error('Either ref or x/y coordinates are required');
+  }
+
+  await client.Input.dispatchMouseEvent({
+    type: 'mouseMoved',
+    x,
+    y
+  });
+
+  return { x, y };
+}