feat: add BiDiServer to drive lightpanda with Selenium over WebDriver BiDi

.github/workflows/wheels.yml

@@ -183,9 +183,10 @@ jobs:
       - name: Install the wheel into a clean venv
         run: |
           uv venv wheel-env
-          # playwright is a CDP client here (connect_over_cdp), so no browser
-          # download (`playwright install`) is needed.
-          uv pip install --python wheel-env dist/*.whl pytest pytest-asyncio playwright
+          # playwright (connect_over_cdp) and selenium (webdriver.Remote) are
+          # only clients of the bundled browser here: no browser or driver
+          # download is needed.
+          uv pip install --python wheel-env dist/*.whl pytest pytest-asyncio playwright selenium
 
       - name: Run the test suite against the installed wheel
         run: |

README.md

@@ -87,6 +87,37 @@ cannot serve MCP and CDP from the same one). The package itself needs only
 Python's standard library; Playwright is a dev-only test dependency and
 `connect_over_cdp` never downloads a browser.
 
+## Drive it with Selenium (WebDriver BiDi)
+
+The browser also speaks [WebDriver BiDi](https://w3c.github.io/webdriver-bidi/).
+`BiDiServer` starts `lightpanda serve --protocol webdriver` and hands you the
+URL Selenium's `webdriver.Remote` takes as `command_executor`:
+
+```python
+from lightpanda import BiDiServer
+from selenium import webdriver
+from selenium.webdriver.common.options import ArgOptions
+
+options = ArgOptions()
+options.web_socket_url = True  # ask for a WebDriver BiDi session
+
+with BiDiServer() as server:
+    driver = webdriver.Remote(command_executor=server.http_endpoint, options=options)
+    context = driver.browsing_context.create(type="tab")
+    driver.browsing_context.navigate(context=context, url="https://example.com", wait="complete")
+    print(driver.script.execute("() => document.title", context_id=context)["value"])
+    driver.quit()
+```
+
+`AsyncBiDiServer` is the asyncio twin, and `server.bidi_endpoint`
+(`ws://127.0.0.1:<port>/session`) is the raw BiDi WebSocket for clients that
+speak the protocol directly. The browser serves the BiDi modules plus the
+classic session bootstrap Selenium needs, not the classic WebDriver
+commands: `driver.get`, `find_element` and friends are not implemented, so
+drive the page through `driver.browsing_context` and `driver.script` with an
+explicit context, created first as above. Pass `args=["--protocol", "cdp"]`
+to serve CDP on the same port as well.
+
 ## How the bindings work
 
 Every browser tool is a `Session` method, typed and documented in your IDE.
@@ -125,8 +156,9 @@ lightpanda binary. Then:
 uv run --group dev pytest tests
 ```
 
-The `dev` group includes `playwright` for the CDP tests (its pip package
-only; no `playwright install`). Those tests skip when it is absent.
+The `dev` group includes `playwright` and `selenium` as clients for the CDP
+and BiDi tests (their pip packages only; no browser or driver download).
+Those tests skip when the client is absent.
 
 Regenerate the tool methods (`lightpanda/_methods.py`) and the API docs:
 

lightpanda/__init__.py

@@ -22,10 +22,12 @@
 
 For Playwright or Puppeteer code, ``CDPServer`` runs the browser's own
 Chrome DevTools Protocol server and hands you the endpoint to connect to
-(see its docs for an example).
+(see its docs for an example). For Selenium, ``BiDiServer`` serves WebDriver
+BiDi the same way and hands you the ``command_executor`` URL.
 """
 
 from .async_browser import AsyncBrowser, AsyncSession, run_script_async
+from .bidi import AsyncBiDiServer, BiDiServer
 from .browser import Browser, Session, run_script
 from .cdp import AsyncCDPServer, CDPServer
 from .errors import LightpandaError, ProtocolError, ScriptError, ToolError
@@ -39,6 +41,8 @@
     "run_script_async",
     "CDPServer",
     "AsyncCDPServer",
+    "BiDiServer",
+    "AsyncBiDiServer",
     "LightpandaError",
     "ProtocolError",
     "ScriptError",

lightpanda/_serve.py

@@ -0,0 +1,149 @@
+"""Shared shape of the ``lightpanda serve`` wrappers (``CDPServer``, ``BiDiServer``).
+
+A serve wrapper owns one ``lightpanda serve`` process on a localhost port
+and hands out endpoints for third-party clients; the subclasses differ only
+in the ``--protocol`` flags they pass and the endpoints they expose.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import urllib.request
+from typing import Generic, TypeVar
+
+from .client import _HOST, _spawn, _terminate, find_binary
+from .errors import LightpandaError
+
+_HTTP_TIMEOUT = 5.0
+
+
+class _ServeProcess:
+    _protocol: tuple[str, ...] = ()  # ``serve`` flags placed before the user's ``args``
+
+    def __init__(
+        self,
+        binary: str | os.PathLike | None = None,
+        env: dict[str, str] | None = None,
+        verbose: bool = False,
+        args: tuple[str, ...] | list[str] = (),
+        port: int | None = None,
+    ):
+        """``port`` pins the listening port (default: a free one). ``args``
+        are extra ``lightpanda serve`` flags; pass ``port=`` rather than
+        ``--port``. ``verbose`` lets the browser log through to stderr."""
+        self._proc, self._port = _spawn(
+            find_binary(binary), "serve", [*self._protocol, *args], env, verbose, port=port
+        )
+
+    @property
+    def port(self) -> int:
+        return self._port
+
+    @property
+    def http_endpoint(self) -> str:
+        """``http://127.0.0.1:<port>``, the server's HTTP root: what Puppeteer
+        (``browserURL``) and Playwright (``connect_over_cdp`` with an http URL)
+        discover the CDP WebSocket from, and Selenium's ``command_executor``."""
+        return f"http://{_HOST}:{self._port}"
+
+    def _get_json(self, path: str) -> dict:
+        """GET ``path`` and parse the JSON body. urllib sends an IP-literal
+        ``Host`` and no ``Origin``, which is what the browser accepts."""
+        if self._proc is None:
+            raise LightpandaError("server closed")
+        url = f"{self.http_endpoint}{path}"
+        try:
+            with urllib.request.urlopen(url, timeout=_HTTP_TIMEOUT) as resp:
+                return json.loads(resp.read())
+        except (OSError, ValueError) as err:  # URLError is an OSError
+            raise LightpandaError(f"GET {url} failed: {err}") from err
+
+    def close(self) -> None:
+        if self._proc is not None:
+            _terminate(self._proc)
+            self._proc = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        self.close()
+
+    def __del__(self):
+        try:
+            self.close()
+        except Exception:
+            pass
+
+
+_S = TypeVar("_S", bound=_ServeProcess)
+
+
+class _AsyncServeProcess(Generic[_S]):
+    """asyncio twin of a :class:`_ServeProcess`: the process is spawned by
+    :meth:`start`, called automatically on ``async with`` entry. Subclasses
+    set ``_sync_cls`` and re-declare their protocol-specific members."""
+
+    _sync_cls: type[_S]
+
+    def __init__(
+        self,
+        binary: str | os.PathLike | None = None,
+        env: dict[str, str] | None = None,
+        verbose: bool = False,
+        args: tuple[str, ...] | list[str] = (),
+        port: int | None = None,
+    ):
+        """Arguments are forwarded to the sync class."""
+        self._kwargs = dict(binary=binary, env=env, verbose=verbose, args=args, port=port)
+        self._server: _S | None = None
+        self._start_lock = asyncio.Lock()
+
+    async def start(self):
+        """Spawn the server process. Idempotent."""
+        if self._server is None:
+            async with self._start_lock:
+                if self._server is None:
+                    self._server = await asyncio.to_thread(self._sync_cls, **self._kwargs)
+        return self
+
+    def _started(self) -> _S:
+        if self._server is None:
+            raise LightpandaError("server not started; use `async with` or `await start()`")
+        return self._server
+
+    @property
+    def port(self) -> int:
+        return self._started().port
+
+    @property
+    def http_endpoint(self) -> str:
+        """``http://127.0.0.1:<port>``, see the sync class."""
+        return self._started().http_endpoint
+
+    async def close(self) -> None:
+        if self._server is not None:
+            server, self._server = self._server, None
+            await asyncio.to_thread(server.close)
+
+    async def __aenter__(self):
+        return await self.start()
+
+    async def __aexit__(self, *exc):
+        await self.close()
+
+
+def _documented(cls: type) -> type:
+    """Copy the public members (and ``__init__``) this module's bases give
+    ``cls`` onto ``cls`` itself, so pdoc and IDEs document them as its own:
+    pdoc hides what is inherited from a private module. Same trick as
+    ``_attach_generated`` in browser.py."""
+    for base in cls.__mro__[1:]:
+        if base.__module__ != __name__:
+            continue
+        for name, member in vars(base).items():
+            if (name == "__init__" or not name.startswith("_")) and name not in vars(cls):
+                setattr(cls, name, member)
+    return cls

lightpanda/bidi.py

@@ -0,0 +1,85 @@
+"""WebDriver BiDi: run ``lightpanda serve --protocol webdriver`` for Selenium and co.
+
+:class:`BiDiServer` spawns the browser's WebDriver BiDi server on a free
+localhost port and owns the process; :class:`AsyncBiDiServer` is the asyncio
+twin. See :class:`BiDiServer` for what the browser serves.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from ._serve import _AsyncServeProcess, _ServeProcess, _documented
+from .client import _HOST
+
+
+@_documented
+class BiDiServer(_ServeProcess):
+    """A lightpanda process serving WebDriver BiDi on 127.0.0.1.
+
+    ```python
+    from lightpanda import BiDiServer
+    from selenium import webdriver
+    from selenium.webdriver.common.options import ArgOptions
+
+    options = ArgOptions()
+    options.web_socket_url = True  # ask for a WebDriver BiDi session
+
+    with BiDiServer() as server:
+        driver = webdriver.Remote(command_executor=server.http_endpoint, options=options)
+        context = driver.browsing_context.create(type="tab")
+        driver.browsing_context.navigate(context=context, url="https://example.com", wait="complete")
+        print(driver.script.execute("() => document.title", context_id=context)["value"])
+        driver.quit()
+    ```
+
+    :attr:`http_endpoint` is Selenium's ``command_executor``. The browser
+    serves the BiDi modules (``session``, ``browser``, ``browsingContext``,
+    ``script``, ``input``) over the WebSocket plus the classic session
+    bootstrap (``GET /status``, ``POST /session`` with the ``webSocketUrl``
+    capability, ``DELETE /session/<id>``); other classic WebDriver commands
+    such as Selenium's ``driver.get`` or ``find_element`` are not served, so
+    drive the page through ``driver.browsing_context`` and ``driver.script``
+    with an explicit context, created first as above. Pass
+    ``args=["--protocol", "cdp"]`` to serve CDP on the same port as well
+    (``--protocol`` is additive). The process is stopped by :meth:`close` /
+    leaving the ``with`` block, and on Linux also when the interpreter dies.
+    """
+
+    _protocol = ("--protocol", "webdriver")
+
+    @property
+    def bidi_endpoint(self) -> str:
+        """The session-less BiDi WebSocket URL, ``ws://127.0.0.1:<port>/session``,
+        for clients that speak BiDi directly (``session.new`` over the socket).
+        A session bootstrapped through ``POST /session`` gets its own socket at
+        ``<bidi_endpoint>/<sessionId>``, returned as the ``webSocketUrl``
+        capability.
+
+        Keep the IP literal: the WebSocket upgrade rejects any ``Origin``
+        header and only accepts an IP-literal or ``localhost`` host."""
+        return f"ws://{_HOST}:{self._port}/session"
+
+    def status(self) -> dict:
+        """The ``GET /status`` value, ``{"ready": True, "message": ""}``."""
+        return self._get_json("/status")["value"]
+
+
+@_documented
+class AsyncBiDiServer(_AsyncServeProcess[BiDiServer]):
+    """:class:`BiDiServer` for asyncio: the process is spawned by
+    :meth:`start`, called automatically on ``async with`` entry."""
+
+    _sync_cls = BiDiServer
+
+    @property
+    def bidi_endpoint(self) -> str:
+        """See :attr:`BiDiServer.bidi_endpoint`."""
+        return self._started().bidi_endpoint
+
+    async def status(self) -> dict:
+        """See :meth:`BiDiServer.status`."""
+        return await asyncio.to_thread(self._started().status)
+
+
+__all__ = ["BiDiServer", "AsyncBiDiServer"]