From d8bea1529c383c712d2e8de9ca36e49d2c6975df Mon Sep 17 00:00:00 2001
From: Mark C Allen <mark@markcallen.com>
Date: Wed, 26 Aug 2026 17:37:38 -0700
Subject: [PATCH 1/3] feat: add ref-based interaction tools and retrofit
 click/type
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add 6 new interaction tools that work with element refs from
browser_snapshot, enabling selector-free workflows:

- browser_fill: replace form field value with React/Vue/Svelte-
  compatible events (native value setter + input/change dispatch)
- browser_hover: move mouse to element by ref or coordinates
- browser_press_key: named keys (Enter, Tab, Escape, Arrow*, F1-F12)
  and modifier combos (Control+a, Meta+c, Shift+Enter)
- browser_select: select <option> by value, label, or index
- browser_check: toggle checkbox/radio with optional target state
- browser_scroll: scroll element into view, by direction, or absolute

Retrofit existing tools:
- browser_click: now accepts ref alongside selector/coordinates
- browser_type: now accepts ref alongside selector

Shared infrastructure:
- ref-resolver: resolve ref → scroll into view → box model coords →
  remote object ID
- interaction/events: framework-compatible value setting and event
  dispatch

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 src/browser/interaction/events.ts       |  79 +++++++++
 src/browser/interaction/ref-resolver.ts |  53 ++++++
 src/commands/check.ts                   |  55 ++++++
 src/commands/click.ts                   |  19 ++-
 src/commands/fill.ts                    |  34 ++++
 src/commands/hover.ts                   |  43 +++++
 src/commands/press_key.ts               | 134 +++++++++++++++
 src/commands/scroll.ts                  |  79 +++++++++
 src/commands/select.ts                  |  74 ++++++++
 src/commands/type.ts                    |  11 +-
 src/mcp/server.ts                       |   2 +
 src/mcp/tools/browser.ts                |  22 ++-
 src/mcp/tools/interaction.ts            | 216 ++++++++++++++++++++++++
 src/mcp/tools/names.ts                  |   8 +-
 14 files changed, 816 insertions(+), 13 deletions(-)
 create mode 100644 src/browser/interaction/events.ts
 create mode 100644 src/browser/interaction/ref-resolver.ts
 create mode 100644 src/commands/check.ts
 create mode 100644 src/commands/fill.ts
 create mode 100644 src/commands/hover.ts
 create mode 100644 src/commands/press_key.ts
 create mode 100644 src/commands/scroll.ts
 create mode 100644 src/commands/select.ts
 create mode 100644 src/mcp/tools/interaction.ts

diff --git a/src/browser/interaction/events.ts b/src/browser/interaction/events.ts
new file mode 100644
index 0000000..f3410b2
--- /dev/null
+++ b/src/browser/interaction/events.ts
@@ -0,0 +1,79 @@
+import type { Client } from 'chrome-remote-interface';
+
+/**
+ * Fire framework-compatible input events on an element.
+ * Uses the native value setter to work with React's synthetic event system,
+ * then dispatches native DOM events with bubbling enabled for Vue/Svelte.
+ */
+export async function fireInputEvents(
+  client: Client,
+  objectId: string
+): Promise<void> {
+  await client.Runtime.callFunctionOn({
+    objectId,
+    functionDeclaration: `function() {
+      this.dispatchEvent(new Event('input', { bubbles: true }));
+      this.dispatchEvent(new Event('change', { bubbles: true }));
+    }`,
+    awaitPromise: false
+  });
+}
+
+/**
+ * Set a form field value using the native setter for React compatibility,
+ * then fire input/change events.
+ */
+export async function setFieldValue(
+  client: Client,
+  objectId: string,
+  value: string
+): Promise<void> {
+  await client.Runtime.callFunctionOn({
+    objectId,
+    functionDeclaration: `function(newValue) {
+      var tag = this.tagName.toLowerCase();
+      if (tag === 'select') {
+        // Select element — set by value
+        for (var i = 0; i < this.options.length; i++) {
+          if (this.options[i].value === newValue) {
+            this.selectedIndex = i;
+            break;
+          }
+        }
+      } else if (this.getAttribute('contenteditable') !== null) {
+        this.textContent = newValue;
+      } else {
+        // Use native setter for React compatibility
+        var proto = tag === 'textarea'
+          ? HTMLTextAreaElement.prototype
+          : HTMLInputElement.prototype;
+        var setter = Object.getOwnPropertyDescriptor(proto, 'value');
+        if (setter && setter.set) {
+          setter.set.call(this, newValue);
+        } else {
+          this.value = newValue;
+        }
+      }
+      this.dispatchEvent(new Event('input', { bubbles: true }));
+      this.dispatchEvent(new Event('change', { bubbles: true }));
+    }`,
+    arguments: [{ value }],
+    awaitPromise: false
+  });
+}
+
+/**
+ * Get the tag name of an element by its remote object ID.
+ */
+export async function getTagName(
+  client: Client,
+  objectId: string
+): Promise<string> {
+  const { result } = await client.Runtime.callFunctionOn({
+    objectId,
+    functionDeclaration: `function() { return this.tagName ? this.tagName.toLowerCase() : ''; }`,
+    returnByValue: true,
+    awaitPromise: false
+  });
+  return String(result.value ?? '');
+}
diff --git a/src/browser/interaction/ref-resolver.ts b/src/browser/interaction/ref-resolver.ts
new file mode 100644
index 0000000..c9af60f
--- /dev/null
+++ b/src/browser/interaction/ref-resolver.ts
@@ -0,0 +1,53 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../inspect/element-ref.js';
+
+export interface ResolvedElement {
+  backendNodeId: number;
+  objectId: string;
+  x: number;
+  y: number;
+}
+
+/**
+ * Resolve an element ref to coordinates and a remote object ID.
+ * Scrolls the element into view first so click coordinates are valid.
+ */
+export async function resolveRef(
+  client: Client,
+  refMap: ElementRefMap,
+  ref: string
+): Promise<ResolvedElement> {
+  const backendNodeId = refMap.resolve(ref);
+
+  // Scroll into view
+  await client.DOM.scrollIntoViewIfNeeded({ backendNodeId });
+
+  // Get center coordinates from box model
+  const { model } = await client.DOM.getBoxModel({ backendNodeId });
+  const content = model.content;
+  const x = (content[0] + content[2] + content[4] + content[6]) / 4;
+  const y = (content[1] + content[3] + content[5] + content[7]) / 4;
+
+  // Get remote object for Runtime.callFunctionOn
+  const { object } = await client.DOM.resolveNode({ backendNodeId });
+  if (!object.objectId) {
+    throw new Error(`Could not resolve ref "${ref}" to a remote object`);
+  }
+
+  return {
+    backendNodeId,
+    objectId: object.objectId,
+    x: Math.round(x),
+    y: Math.round(y)
+  };
+}
+
+/**
+ * Focus an element by ref.
+ */
+export async function focusRef(
+  client: Client,
+  backendNodeId: number
+): Promise<void> {
+  await client.DOM.focus({ backendNodeId });
+}
diff --git a/src/commands/check.ts b/src/commands/check.ts
new file mode 100644
index 0000000..46df2ae
--- /dev/null
+++ b/src/commands/check.ts
@@ -0,0 +1,55 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { resolveRef } from '../browser/interaction/ref-resolver.js';
+
+export interface CheckParams {
+  ref: string;
+  checked?: boolean;
+}
+
+export interface CheckResult {
+  ref: string;
+  checked: boolean;
+}
+
+/**
+ * Toggle a checkbox or radio button by ref.
+ * If `checked` is specified, sets it to that state (no-op if already matching).
+ * If omitted, toggles the current state.
+ */
+export async function executeCheck(
+  client: Client,
+  refMap: ElementRefMap,
+  params: CheckParams
+): Promise<CheckResult> {
+  const resolved = await resolveRef(client, refMap, params.ref);
+
+  const { result } = await client.Runtime.callFunctionOn({
+    objectId: resolved.objectId,
+    functionDeclaration: `function(desiredState) {
+      var tag = this.tagName.toLowerCase();
+      var type = (this.type || '').toLowerCase();
+      if (tag !== 'input' || (type !== 'checkbox' && type !== 'radio')) {
+        throw new Error('Element is not a checkbox or radio button');
+      }
+      var current = this.checked;
+      var target = desiredState !== null ? desiredState : !current;
+      if (current !== target) {
+        this.click();
+      }
+      return this.checked;
+    }`,
+    arguments: [{ value: params.checked ?? null }],
+    returnByValue: true,
+    awaitPromise: false
+  });
+
+  if (result.subtype === 'error' || result.className === 'Error') {
+    throw new Error(result.description ?? 'Failed to toggle checkbox');
+  }
+
+  return {
+    ref: params.ref,
+    checked: result.value as boolean
+  };
+}
diff --git a/src/commands/click.ts b/src/commands/click.ts
index daef536..932dbad 100644
--- a/src/commands/click.ts
+++ b/src/commands/click.ts
@@ -1,8 +1,11 @@
 import { z } from 'zod';
 import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { resolveRef } from '../browser/interaction/ref-resolver.js';
 
 export const ClickParamsSchema = z
   .object({
+    ref: z.string().optional(),
     selector: z.string().optional(),
     x: z.number().optional(),
     y: z.number().optional(),
@@ -11,9 +14,12 @@ export const ClickParamsSchema = z
     timeoutMs: z.number().int().positive().default(5000)
   })
   .refine(
-    (v) => v.selector !== undefined || (v.x !== undefined && v.y !== undefined),
+    (v) =>
+      v.ref !== undefined ||
+      v.selector !== undefined ||
+      (v.x !== undefined && v.y !== undefined),
     {
-      message: 'Either selector or x/y coordinates are required'
+      message: 'Either ref, selector, or x/y coordinates are required'
     }
   );
 
@@ -52,12 +58,17 @@ async function waitForSelector(
 
 export async function executeClick(
   client: Client,
-  params: ClickParams
+  params: ClickParams,
+  refMap?: ElementRefMap
 ): Promise<ClickResult> {
   let x: number;
   let y: number;
 
-  if (params.selector !== undefined) {
+  if (params.ref !== undefined && refMap) {
+    const resolved = await resolveRef(client, refMap, params.ref);
+    x = resolved.x;
+    y = resolved.y;
+  } else if (params.selector !== undefined) {
     const pos = await waitForSelector(
       client,
       params.selector,
diff --git a/src/commands/fill.ts b/src/commands/fill.ts
new file mode 100644
index 0000000..bff5812
--- /dev/null
+++ b/src/commands/fill.ts
@@ -0,0 +1,34 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { resolveRef, focusRef } from '../browser/interaction/ref-resolver.js';
+import { setFieldValue } from '../browser/interaction/events.js';
+
+export interface FillParams {
+  ref: string;
+  value: string;
+}
+
+export interface FillResult {
+  ref: string;
+  value: string;
+}
+
+/**
+ * Fill a form field by ref — clears the existing value and sets a new one.
+ * Fires framework-compatible input/change events.
+ */
+export async function executeFill(
+  client: Client,
+  refMap: ElementRefMap,
+  params: FillParams
+): Promise<FillResult> {
+  const resolved = await resolveRef(client, refMap, params.ref);
+
+  // Focus the element
+  await focusRef(client, resolved.backendNodeId);
+
+  // Set value and fire events
+  await setFieldValue(client, resolved.objectId, params.value);
+
+  return { ref: params.ref, value: params.value };
+}
diff --git a/src/commands/hover.ts b/src/commands/hover.ts
new file mode 100644
index 0000000..178ca38
--- /dev/null
+++ b/src/commands/hover.ts
@@ -0,0 +1,43 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { resolveRef } from '../browser/interaction/ref-resolver.js';
+
+export interface HoverParams {
+  ref?: string;
+  selector?: string;
+  x?: number;
+  y?: number;
+}
+
+export interface HoverResult {
+  x: number;
+  y: number;
+}
+
+export async function executeHover(
+  client: Client,
+  refMap: ElementRefMap,
+  params: HoverParams
+): Promise<HoverResult> {
+  let x: number;
+  let y: number;
+
+  if (params.ref) {
+    const resolved = await resolveRef(client, refMap, params.ref);
+    x = resolved.x;
+    y = resolved.y;
+  } else if (params.x !== undefined && params.y !== undefined) {
+    x = params.x;
+    y = params.y;
+  } else {
+    throw new Error('Either ref or x/y coordinates are required');
+  }
+
+  await client.Input.dispatchMouseEvent({
+    type: 'mouseMoved',
+    x,
+    y
+  });
+
+  return { x, y };
+}
diff --git a/src/commands/press_key.ts b/src/commands/press_key.ts
new file mode 100644
index 0000000..7c6dc82
--- /dev/null
+++ b/src/commands/press_key.ts
@@ -0,0 +1,134 @@
+import type { Client } from 'chrome-remote-interface';
+
+/**
+ * Map of named keys to CDP key event properties.
+ */
+const KEY_MAP: Record<
+  string,
+  { key: string; code: string; keyCode?: number; text?: string }
+> = {
+  enter: { key: 'Enter', code: 'Enter', keyCode: 13, text: '\r' },
+  tab: { key: 'Tab', code: 'Tab', keyCode: 9 },
+  escape: { key: 'Escape', code: 'Escape', keyCode: 27 },
+  backspace: { key: 'Backspace', code: 'Backspace', keyCode: 8 },
+  delete: { key: 'Delete', code: 'Delete', keyCode: 46 },
+  space: { key: ' ', code: 'Space', keyCode: 32, text: ' ' },
+  arrowup: { key: 'ArrowUp', code: 'ArrowUp', keyCode: 38 },
+  arrowdown: { key: 'ArrowDown', code: 'ArrowDown', keyCode: 40 },
+  arrowleft: { key: 'ArrowLeft', code: 'ArrowLeft', keyCode: 37 },
+  arrowright: { key: 'ArrowRight', code: 'ArrowRight', keyCode: 39 },
+  home: { key: 'Home', code: 'Home', keyCode: 36 },
+  end: { key: 'End', code: 'End', keyCode: 35 },
+  pageup: { key: 'PageUp', code: 'PageUp', keyCode: 33 },
+  pagedown: { key: 'PageDown', code: 'PageDown', keyCode: 34 },
+  f1: { key: 'F1', code: 'F1', keyCode: 112 },
+  f2: { key: 'F2', code: 'F2', keyCode: 113 },
+  f3: { key: 'F3', code: 'F3', keyCode: 114 },
+  f4: { key: 'F4', code: 'F4', keyCode: 115 },
+  f5: { key: 'F5', code: 'F5', keyCode: 116 },
+  f6: { key: 'F6', code: 'F6', keyCode: 117 },
+  f7: { key: 'F7', code: 'F7', keyCode: 118 },
+  f8: { key: 'F8', code: 'F8', keyCode: 119 },
+  f9: { key: 'F9', code: 'F9', keyCode: 120 },
+  f10: { key: 'F10', code: 'F10', keyCode: 121 },
+  f11: { key: 'F11', code: 'F11', keyCode: 122 },
+  f12: { key: 'F12', code: 'F12', keyCode: 123 }
+};
+
+// CDP modifier bit flags
+const MODIFIER_FLAGS: Record<string, number> = {
+  alt: 1,
+  control: 2,
+  ctrl: 2,
+  meta: 4,
+  cmd: 4,
+  command: 4,
+  shift: 8
+};
+
+export interface PressKeyParams {
+  key: string;
+}
+
+export interface PressKeyResult {
+  key: string;
+  modifiers: string[];
+}
+
+/**
+ * Parse a key string like "Control+a", "Meta+Shift+Enter", or "Tab".
+ */
+function parseKeyCombo(input: string): {
+  modifiers: number;
+  modifierNames: string[];
+  keyInfo: { key: string; code: string; keyCode?: number; text?: string };
+} {
+  const parts = input.split('+');
+  let modifiers = 0;
+  const modifierNames: string[] = [];
+
+  // Last part is the actual key, everything before is a modifier
+  const keyPart = parts.pop()!;
+
+  for (const mod of parts) {
+    const flag = MODIFIER_FLAGS[mod.toLowerCase()];
+    if (flag) {
+      modifiers |= flag;
+      modifierNames.push(mod);
+    }
+  }
+
+  // Look up the key in our map
+  const mapped = KEY_MAP[keyPart.toLowerCase()];
+  if (mapped) {
+    return { modifiers, modifierNames, keyInfo: mapped };
+  }
+
+  // Single character key
+  if (keyPart.length === 1) {
+    const charCode = keyPart.charCodeAt(0);
+    return {
+      modifiers,
+      modifierNames,
+      keyInfo: {
+        key: keyPart,
+        code: `Key${keyPart.toUpperCase()}`,
+        keyCode: charCode,
+        text: modifiers === 0 ? keyPart : undefined
+      }
+    };
+  }
+
+  // Unknown named key — pass through
+  return {
+    modifiers,
+    modifierNames,
+    keyInfo: { key: keyPart, code: keyPart }
+  };
+}
+
+export async function executePressKey(
+  client: Client,
+  params: PressKeyParams
+): Promise<PressKeyResult> {
+  const { modifiers, modifierNames, keyInfo } = parseKeyCombo(params.key);
+
+  await client.Input.dispatchKeyEvent({
+    type: 'keyDown',
+    key: keyInfo.key,
+    code: keyInfo.code,
+    windowsVirtualKeyCode: keyInfo.keyCode,
+    modifiers,
+    ...(keyInfo.text && { text: keyInfo.text })
+  });
+
+  await client.Input.dispatchKeyEvent({
+    type: 'keyUp',
+    key: keyInfo.key,
+    code: keyInfo.code,
+    windowsVirtualKeyCode: keyInfo.keyCode,
+    modifiers
+  });
+
+  return { key: keyInfo.key, modifiers: modifierNames };
+}
diff --git a/src/commands/scroll.ts b/src/commands/scroll.ts
new file mode 100644
index 0000000..fb88046
--- /dev/null
+++ b/src/commands/scroll.ts
@@ -0,0 +1,79 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+
+export interface ScrollParams {
+  ref?: string;
+  x?: number;
+  y?: number;
+  direction?: 'up' | 'down' | 'left' | 'right';
+  amount?: number;
+}
+
+export interface ScrollResult {
+  scrollX: number;
+  scrollY: number;
+}
+
+/**
+ * Scroll the page or an element.
+ * - If ref is given, scrolls the element into view.
+ * - If direction/amount is given, scrolls the page.
+ * - If x/y is given, scrolls the page to absolute coordinates.
+ */
+export async function executeScroll(
+  client: Client,
+  refMap: ElementRefMap,
+  params: ScrollParams
+): Promise<ScrollResult> {
+  if (params.ref) {
+    // Scroll element into view
+    const backendNodeId = refMap.resolve(params.ref);
+    await client.DOM.scrollIntoViewIfNeeded({ backendNodeId });
+
+    const { result } = await client.Runtime.evaluate({
+      expression: `JSON.stringify({ scrollX: window.scrollX, scrollY: window.scrollY })`,
+      returnByValue: true
+    });
+    const pos = JSON.parse(String(result.value));
+    return { scrollX: pos.scrollX, scrollY: pos.scrollY };
+  }
+
+  if (params.direction) {
+    const amount = params.amount ?? 300;
+    const deltaX =
+      params.direction === 'left'
+        ? -amount
+        : params.direction === 'right'
+          ? amount
+          : 0;
+    const deltaY =
+      params.direction === 'up'
+        ? -amount
+        : params.direction === 'down'
+          ? amount
+          : 0;
+
+    await client.Input.dispatchMouseEvent({
+      type: 'mouseWheel',
+      x: 0,
+      y: 0,
+      deltaX,
+      deltaY
+    });
+
+    // Small delay for scroll to take effect
+    await new Promise((r) => setTimeout(r, 100));
+  } else if (params.x !== undefined || params.y !== undefined) {
+    await client.Runtime.evaluate({
+      expression: `window.scrollTo(${params.x ?? 0}, ${params.y ?? 0})`,
+      awaitPromise: false
+    });
+  }
+
+  const { result } = await client.Runtime.evaluate({
+    expression: `JSON.stringify({ scrollX: window.scrollX, scrollY: window.scrollY })`,
+    returnByValue: true
+  });
+  const pos = JSON.parse(String(result.value));
+  return { scrollX: pos.scrollX, scrollY: pos.scrollY };
+}
diff --git a/src/commands/select.ts b/src/commands/select.ts
new file mode 100644
index 0000000..afa4b6b
--- /dev/null
+++ b/src/commands/select.ts
@@ -0,0 +1,74 @@
+import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { resolveRef, focusRef } from '../browser/interaction/ref-resolver.js';
+
+export interface SelectParams {
+  ref: string;
+  value?: string;
+  label?: string;
+  index?: number;
+}
+
+export interface SelectResult {
+  ref: string;
+  selectedValue: string;
+  selectedLabel: string;
+}
+
+/**
+ * Select an option in a <select> element by value, label, or index.
+ */
+export async function executeSelect(
+  client: Client,
+  refMap: ElementRefMap,
+  params: SelectParams
+): Promise<SelectResult> {
+  const resolved = await resolveRef(client, refMap, params.ref);
+
+  await focusRef(client, resolved.backendNodeId);
+
+  const { result } = await client.Runtime.callFunctionOn({
+    objectId: resolved.objectId,
+    functionDeclaration: `function(byValue, byLabel, byIndex) {
+      if (this.tagName.toLowerCase() !== 'select') {
+        throw new Error('Element is not a <select>');
+      }
+      var found = false;
+      for (var i = 0; i < this.options.length; i++) {
+        var opt = this.options[i];
+        if (byValue !== null && opt.value === byValue) {
+          this.selectedIndex = i; found = true; break;
+        }
+        if (byLabel !== null && opt.textContent.trim() === byLabel) {
+          this.selectedIndex = i; found = true; break;
+        }
+        if (byIndex !== null && i === byIndex) {
+          this.selectedIndex = i; found = true; break;
+        }
+      }
+      if (!found) throw new Error('Option not found');
+      this.dispatchEvent(new Event('input', { bubbles: true }));
+      this.dispatchEvent(new Event('change', { bubbles: true }));
+      var sel = this.options[this.selectedIndex];
+      return { value: sel.value, label: sel.textContent.trim() };
+    }`,
+    arguments: [
+      { value: params.value ?? null },
+      { value: params.label ?? null },
+      { value: params.index ?? null }
+    ],
+    returnByValue: true,
+    awaitPromise: false
+  });
+
+  if (result.subtype === 'error' || result.className === 'Error') {
+    throw new Error(result.description ?? 'Failed to select option');
+  }
+
+  const selected = result.value as { value: string; label: string };
+  return {
+    ref: params.ref,
+    selectedValue: selected.value,
+    selectedLabel: selected.label
+  };
+}
diff --git a/src/commands/type.ts b/src/commands/type.ts
index a68502b..07eea66 100644
--- a/src/commands/type.ts
+++ b/src/commands/type.ts
@@ -1,8 +1,11 @@
 import { z } from 'zod';
 import type { Client } from 'chrome-remote-interface';
+import type { ElementRefMap } from '../browser/inspect/element-ref.js';
+import { resolveRef, focusRef } from '../browser/interaction/ref-resolver.js';
 
 export const TypeParamsSchema = z.object({
   text: z.string(),
+  ref: z.string().optional(),
   selector: z.string().optional(),
   clearFirst: z.boolean().default(false),
   delayMs: z.number().int().min(0).default(0)
@@ -30,9 +33,13 @@ async function focusSelector(client: Client, selector: string): Promise<void> {
 
 export async function executeType(
   client: Client,
-  params: TypeParams
+  params: TypeParams,
+  refMap?: ElementRefMap
 ): Promise<TypeResult> {
-  if (params.selector) {
+  if (params.ref && refMap) {
+    const resolved = await resolveRef(client, refMap, params.ref);
+    await focusRef(client, resolved.backendNodeId);
+  } else if (params.selector) {
     await focusSelector(client, params.selector);
   }
 
diff --git a/src/mcp/server.ts b/src/mcp/server.ts
index ee7afe5..0aab8d6 100644
--- a/src/mcp/server.ts
+++ b/src/mcp/server.ts
@@ -6,6 +6,7 @@ import { ConsoleBuffer, type ConsoleMessage } from './console-buffer.js';
 import { registerBrowserTools } from './tools/browser.js';
 import { registerErrorTools } from './tools/errors.js';
 import { registerSnapshotTools } from './tools/snapshot.js';
+import { registerInteractionTools } from './tools/interaction.js';
 
 export interface McpConfig {
   bufferSize: number;
@@ -63,6 +64,7 @@ export async function createMcpServer(config: McpConfig): Promise<{
   registerBrowserTools(server, context, makeBrowserManager);
   registerErrorTools(server, consoleBuffer);
   registerSnapshotTools(server, context);
+  registerInteractionTools(server, context);
 
   return {
     server,
diff --git a/src/mcp/tools/browser.ts b/src/mcp/tools/browser.ts
index 99cbd3b..9070050 100644
--- a/src/mcp/tools/browser.ts
+++ b/src/mcp/tools/browser.ts
@@ -57,6 +57,10 @@ const navigateShape = {
 };
 
 const clickShape = {
+  ref: z
+    .string()
+    .optional()
+    .describe('Element ref from browser_snapshot (e.g. "e3")'),
   selector: z.string().optional().describe('CSS selector to click'),
   x: z.number().optional().describe('Viewport X coordinate'),
   y: z.number().optional().describe('Viewport Y coordinate'),
@@ -80,6 +84,10 @@ const clickShape = {
 
 const typeShape = {
   text: z.string().describe('Text to type'),
+  ref: z
+    .string()
+    .optional()
+    .describe('Element ref from browser_snapshot (e.g. "e3")'),
   selector: z.string().optional().describe('CSS selector to focus first'),
   clearFirst: z
     .boolean()
@@ -361,10 +369,11 @@ export function registerBrowserTools(
 
   server.tool(
     'browser_click',
-    'Click an element by CSS selector or coordinates. Either selector or x/y coordinates required.',
+    'Click an element by ref (from browser_snapshot), CSS selector, or coordinates.',
     clickShape,
     async (params) => {
       if (
+        params.ref === undefined &&
         params.selector === undefined &&
         (params.x === undefined || params.y === undefined)
       ) {
@@ -372,7 +381,7 @@ export function registerBrowserTools(
           content: [
             {
               type: 'text' as const,
-              text: 'Error: Either selector or x/y coordinates are required'
+              text: 'Error: Either ref, selector, or x/y coordinates are required'
             }
           ],
           isError: true
@@ -380,7 +389,7 @@ export function registerBrowserTools(
       }
 
       const { client } = requireClient(context);
-      const result = await executeClick(client, params);
+      const result = await executeClick(client, params, context.elementRefMap);
 
       return {
         content: [
@@ -395,17 +404,18 @@ export function registerBrowserTools(
 
   server.tool(
     'browser_type',
-    'Type text into the focused element or a specified selector',
+    'Type text into the focused element, a ref (from browser_snapshot), or a CSS selector. Generates keystroke events (use browser_fill to replace a field value entirely).',
     typeShape,
     async (params) => {
       const { client } = requireClient(context);
-      await executeType(client, params);
+      await executeType(client, params, context.elementRefMap);
 
+      const target = params.ref ?? params.selector;
       return {
         content: [
           {
             type: 'text' as const,
-            text: `Typed "${params.text}"${params.selector ? ` into ${params.selector}` : ''}`
+            text: `Typed "${params.text}"${target ? ` into ${target}` : ''}`
           }
         ]
       };
diff --git a/src/mcp/tools/interaction.ts b/src/mcp/tools/interaction.ts
new file mode 100644
index 0000000..8489328
--- /dev/null
+++ b/src/mcp/tools/interaction.ts
@@ -0,0 +1,216 @@
+import { z } from 'zod';
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { BrowserContext } from '../server.js';
+import { executeFill } from '../../commands/fill.js';
+import { executeHover } from '../../commands/hover.js';
+import { executePressKey } from '../../commands/press_key.js';
+import { executeSelect } from '../../commands/select.js';
+import { executeCheck } from '../../commands/check.js';
+import { executeScroll } from '../../commands/scroll.js';
+
+function requireContext(context: BrowserContext) {
+  const manager = context.manager;
+  if (!manager || !manager.isConnected()) {
+    throw new Error('Browser not started. Call browser_start first.');
+  }
+  const client = manager.getClient()!;
+  return { client, refMap: context.elementRefMap };
+}
+
+const fillShape = {
+  ref: z.string().describe('Element ref from browser_snapshot (e.g. "e3")'),
+  value: z.string().describe('Value to set in the field')
+};
+
+const hoverShape = {
+  ref: z
+    .string()
+    .optional()
+    .describe('Element ref from browser_snapshot (e.g. "e3")'),
+  x: z.number().optional().describe('Viewport X coordinate'),
+  y: z.number().optional().describe('Viewport Y coordinate')
+};
+
+const pressKeyShape = {
+  key: z
+    .string()
+    .describe(
+      'Key to press. Named keys: Enter, Tab, Escape, Backspace, Delete, Space, ArrowUp/Down/Left/Right, Home, End, PageUp/Down, F1-F12. Modifier combos: Control+a, Meta+c, Shift+Enter.'
+    )
+};
+
+const selectShape = {
+  ref: z.string().describe('Element ref of a <select> element'),
+  value: z.string().optional().describe('Option value attribute to select'),
+  label: z.string().optional().describe('Option visible text to select'),
+  index: z
+    .number()
+    .int()
+    .min(0)
+    .optional()
+    .describe('Option index to select (0-based)')
+};
+
+const checkShape = {
+  ref: z.string().describe('Element ref of a checkbox or radio button'),
+  checked: z.boolean().optional().describe('Desired state. Omit to toggle.')
+};
+
+const scrollShape = {
+  ref: z.string().optional().describe('Element ref to scroll into view'),
+  direction: z
+    .enum(['up', 'down', 'left', 'right'])
+    .optional()
+    .describe('Scroll direction'),
+  amount: z
+    .number()
+    .int()
+    .positive()
+    .optional()
+    .describe('Scroll amount in pixels (default: 300)'),
+  x: z.number().optional().describe('Absolute scroll X position'),
+  y: z.number().optional().describe('Absolute scroll Y position')
+};
+
+export function registerInteractionTools(
+  server: McpServer,
+  context: BrowserContext
+): void {
+  server.tool(
+    'browser_fill',
+    'Fill a form field by element ref. Replaces the entire field value and fires input/change events compatible with React, Vue, and Svelte. Use browser_snapshot first to get refs.',
+    fillShape,
+    async ({ ref, value }) => {
+      const { client, refMap } = requireContext(context);
+      const result = await executeFill(client, refMap, { ref, value });
+      return {
+        content: [
+          {
+            type: 'text' as const,
+            text: `Filled ${result.ref} with "${result.value}"`
+          }
+        ]
+      };
+    }
+  );
+
+  server.tool(
+    'browser_hover',
+    'Hover over an element by ref or coordinates. Dispatches a mouseMoved event.',
+    hoverShape,
+    async (params) => {
+      if (
+        params.ref === undefined &&
+        (params.x === undefined || params.y === undefined)
+      ) {
+        return {
+          content: [
+            {
+              type: 'text' as const,
+              text: 'Error: Either ref or x/y coordinates are required'
+            }
+          ],
+          isError: true
+        };
+      }
+      const { client, refMap } = requireContext(context);
+      const result = await executeHover(client, refMap, params);
+      return {
+        content: [
+          {
+            type: 'text' as const,
+            text: `Hovered at (${result.x}, ${result.y})`
+          }
+        ]
+      };
+    }
+  );
+
+  server.tool(
+    'browser_press_key',
+    'Press a key or key combination. Supports named keys (Enter, Tab, Escape, etc.) and modifier combos (Control+a, Meta+c, Shift+Enter).',
+    pressKeyShape,
+    async ({ key }) => {
+      const { client } = requireContext(context);
+      const result = await executePressKey(client, { key });
+      const modStr =
+        result.modifiers.length > 0
+          ? ` with modifiers: ${result.modifiers.join('+')}`
+          : '';
+      return {
+        content: [
+          { type: 'text' as const, text: `Pressed ${result.key}${modStr}` }
+        ]
+      };
+    }
+  );
+
+  server.tool(
+    'browser_select',
+    'Select an option in a <select> element by value, label, or index. Use browser_snapshot to get the ref.',
+    selectShape,
+    async (params) => {
+      if (
+        params.value === undefined &&
+        params.label === undefined &&
+        params.index === undefined
+      ) {
+        return {
+          content: [
+            {
+              type: 'text' as const,
+              text: 'Error: One of value, label, or index is required'
+            }
+          ],
+          isError: true
+        };
+      }
+      const { client, refMap } = requireContext(context);
+      const result = await executeSelect(client, refMap, params);
+      return {
+        content: [
+          {
+            type: 'text' as const,
+            text: `Selected "${result.selectedLabel}" (value="${result.selectedValue}") in ${result.ref}`
+          }
+        ]
+      };
+    }
+  );
+
+  server.tool(
+    'browser_check',
+    'Toggle a checkbox or radio button by ref. Optionally set to a specific checked state.',
+    checkShape,
+    async ({ ref, checked }) => {
+      const { client, refMap } = requireContext(context);
+      const result = await executeCheck(client, refMap, { ref, checked });
+      return {
+        content: [
+          {
+            type: 'text' as const,
+            text: `${result.ref} is now ${result.checked ? 'checked' : 'unchecked'}`
+          }
+        ]
+      };
+    }
+  );
+
+  server.tool(
+    'browser_scroll',
+    'Scroll the page or an element. Use ref to scroll an element into view, direction/amount for relative scrolling, or x/y for absolute scroll position.',
+    scrollShape,
+    async (params) => {
+      const { client, refMap } = requireContext(context);
+      const result = await executeScroll(client, refMap, params);
+      return {
+        content: [
+          {
+            type: 'text' as const,
+            text: `Scrolled to (${result.scrollX}, ${result.scrollY})`
+          }
+        ]
+      };
+    }
+  );
+}
diff --git a/src/mcp/tools/names.ts b/src/mcp/tools/names.ts
index 441d2b0..2322b13 100644
--- a/src/mcp/tools/names.ts
+++ b/src/mcp/tools/names.ts
@@ -21,7 +21,13 @@ export const EXPECTED_TOOLS = [
   'browser_get_errors',
   'browser_clear_errors',
   'browser_snapshot',
-  'browser_find'
+  'browser_find',
+  'browser_fill',
+  'browser_hover',
+  'browser_press_key',
+  'browser_select',
+  'browser_check',
+  'browser_scroll'
 ] as const;
 
 export type ToolName = (typeof EXPECTED_TOOLS)[number];

From 8329b5a0b2991ea430309d72ff7b00fae4f796ff Mon Sep 17 00:00:00 2001
From: Mark C Allen <mark@markcallen.com>
Date: Wed, 26 Aug 2026 17:46:58 -0700
Subject: [PATCH 2/3] fix: update e2e test to match new click error message

The click tool now accepts ref in addition to selector/coordinates,
so the error message changed from "selector or x/y" to
"ref, selector, or x/y".

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 e2e/mcp/browser-tools.test.ts | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/e2e/mcp/browser-tools.test.ts b/e2e/mcp/browser-tools.test.ts
index 0edcb61..f1dbe94 100644
--- a/e2e/mcp/browser-tools.test.ts
+++ b/e2e/mcp/browser-tools.test.ts
@@ -292,9 +292,11 @@ describe('MCP E2E: browser control tools', () => {
       expect(mcp.getText(result)).toMatch(/clicked at/i);
     });
 
-    it('returns an error when neither selector nor coordinates are provided', async () => {
+    it('returns an error when neither ref, selector, nor coordinates are provided', async () => {
       const result = await mcp.callTool('browser_click', {});
-      expect(mcp.getText(result)).toMatch(/selector or x\/y/i);
+      expect(mcp.getText(result)).toMatch(
+        /ref.*selector.*x\/y|selector.*x\/y/i
+      );
     });
   });
 });

From 94924b3483292c4ca1c6fa4e9b433abfde76ad43 Mon Sep 17 00:00:00 2001
From: Mark C Allen <mark@markcallen.com>
Date: Wed, 26 Aug 2026 18:19:52 -0700
Subject: [PATCH 3/3] fix: address Copilot review feedback on interaction tools

- click/type: fail fast when ref provided without refMap instead of
  silently falling through to undefined coordinates
- hover: remove unused selector field from HoverParams
- scroll: validate that at least one of ref/direction/x/y is provided
- check/select: use exceptionDetails from Runtime.callFunctionOn
  for reliable error reporting instead of checking result.subtype

E2E tests for new interaction tools are planned for Phase 11.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 src/commands/check.ts        | 10 +++++++---
 src/commands/click.ts        |  5 ++++-
 src/commands/hover.ts        |  1 -
 src/commands/select.ts       | 10 +++++++---
 src/commands/type.ts         |  5 ++++-
 src/mcp/tools/interaction.ts | 16 ++++++++++++++++
 6 files changed, 38 insertions(+), 9 deletions(-)

diff --git a/src/commands/check.ts b/src/commands/check.ts
index 46df2ae..4494e7b 100644
--- a/src/commands/check.ts
+++ b/src/commands/check.ts
@@ -24,7 +24,7 @@ export async function executeCheck(
 ): Promise<CheckResult> {
   const resolved = await resolveRef(client, refMap, params.ref);
 
-  const { result } = await client.Runtime.callFunctionOn({
+  const { result, exceptionDetails } = await client.Runtime.callFunctionOn({
     objectId: resolved.objectId,
     functionDeclaration: `function(desiredState) {
       var tag = this.tagName.toLowerCase();
@@ -44,8 +44,12 @@ export async function executeCheck(
     awaitPromise: false
   });
 
-  if (result.subtype === 'error' || result.className === 'Error') {
-    throw new Error(result.description ?? 'Failed to toggle checkbox');
+  if (exceptionDetails) {
+    throw new Error(
+      exceptionDetails.exception?.description ??
+        exceptionDetails.text ??
+        'Failed to toggle checkbox'
+    );
   }
 
   return {
diff --git a/src/commands/click.ts b/src/commands/click.ts
index 932dbad..223f03f 100644
--- a/src/commands/click.ts
+++ b/src/commands/click.ts
@@ -64,7 +64,10 @@ export async function executeClick(
   let x: number;
   let y: number;
 
-  if (params.ref !== undefined && refMap) {
+  if (params.ref !== undefined) {
+    if (!refMap) {
+      throw new Error('Element ref map is required when using ref parameter');
+    }
     const resolved = await resolveRef(client, refMap, params.ref);
     x = resolved.x;
     y = resolved.y;
diff --git a/src/commands/hover.ts b/src/commands/hover.ts
index 178ca38..0f8acad 100644
--- a/src/commands/hover.ts
+++ b/src/commands/hover.ts
@@ -4,7 +4,6 @@ import { resolveRef } from '../browser/interaction/ref-resolver.js';
 
 export interface HoverParams {
   ref?: string;
-  selector?: string;
   x?: number;
   y?: number;
 }
diff --git a/src/commands/select.ts b/src/commands/select.ts
index afa4b6b..cba2b79 100644
--- a/src/commands/select.ts
+++ b/src/commands/select.ts
@@ -27,7 +27,7 @@ export async function executeSelect(
 
   await focusRef(client, resolved.backendNodeId);
 
-  const { result } = await client.Runtime.callFunctionOn({
+  const { result, exceptionDetails } = await client.Runtime.callFunctionOn({
     objectId: resolved.objectId,
     functionDeclaration: `function(byValue, byLabel, byIndex) {
       if (this.tagName.toLowerCase() !== 'select') {
@@ -61,8 +61,12 @@ export async function executeSelect(
     awaitPromise: false
   });
 
-  if (result.subtype === 'error' || result.className === 'Error') {
-    throw new Error(result.description ?? 'Failed to select option');
+  if (exceptionDetails) {
+    throw new Error(
+      exceptionDetails.exception?.description ??
+        exceptionDetails.text ??
+        'Failed to select option'
+    );
   }
 
   const selected = result.value as { value: string; label: string };
diff --git a/src/commands/type.ts b/src/commands/type.ts
index 07eea66..7113c0c 100644
--- a/src/commands/type.ts
+++ b/src/commands/type.ts
@@ -36,7 +36,10 @@ export async function executeType(
   params: TypeParams,
   refMap?: ElementRefMap
 ): Promise<TypeResult> {
-  if (params.ref && refMap) {
+  if (params.ref) {
+    if (!refMap) {
+      throw new Error('Element ref map is required when using ref parameter');
+    }
     const resolved = await resolveRef(client, refMap, params.ref);
     await focusRef(client, resolved.backendNodeId);
   } else if (params.selector) {
diff --git a/src/mcp/tools/interaction.ts b/src/mcp/tools/interaction.ts
index 8489328..6bef0f8 100644
--- a/src/mcp/tools/interaction.ts
+++ b/src/mcp/tools/interaction.ts
@@ -201,6 +201,22 @@ export function registerInteractionTools(
     'Scroll the page or an element. Use ref to scroll an element into view, direction/amount for relative scrolling, or x/y for absolute scroll position.',
     scrollShape,
     async (params) => {
+      if (
+        params.ref === undefined &&
+        params.direction === undefined &&
+        params.x === undefined &&
+        params.y === undefined
+      ) {
+        return {
+          content: [
+            {
+              type: 'text' as const,
+              text: 'Error: At least one of ref, direction, or x/y is required'
+            }
+          ],
+          isError: true
+        };
+      }
       const { client, refMap } = requireContext(context);
       const result = await executeScroll(client, refMap, params);
       return {