diff --git a/.github/workflows/python-reference.yml b/.github/workflows/python-reference.yml new file mode 100644 index 0000000..c8bb612 --- /dev/null +++ b/.github/workflows/python-reference.yml @@ -0,0 +1,69 @@ +name: python-reference + +# Regenerate the Python SDK API reference page from the lightpanda-python main +# branch and open a pull request when the output changed. The page is +# src/content/reference/python-api.mdx, written by +# scripts/generate-python-reference.py with pdoc's Python API, so it renders +# like any other docs page and goes live at +# https://lightpanda.io/docs/reference/python-api once the website bumps its +# docs submodule like any other docs change. The package imports without a +# browser binary, so none is needed here. Runs daily to pick up new package +# changes, by hand, or as a smoke run when the generator itself changes. + +on: + schedule: + - cron: "17 6 * * *" + workflow_dispatch: + pull_request: + paths: + - scripts/generate-python-reference.py + - .github/workflows/python-reference.yml + +permissions: + contents: write + pull-requests: write + +concurrency: + group: python-reference + cancel-in-progress: true + +jobs: + update: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + + - name: Generate the reference page + id: generate + run: | + sha=$(git ls-remote https://github.com/lightpanda-io/lightpanda-python.git refs/heads/main | cut -f1) + uv run --no-project --python 3.13 --with 'pdoc==16.0.0' \ + --with "git+https://github.com/lightpanda-io/lightpanda-python@$sha" \ + python scripts/generate-python-reference.py + echo "sha=${sha::7}" >> "$GITHUB_OUTPUT" + git status --short src/content/reference/python-api.mdx + + - name: Open a pull request if the reference changed + if: github.event_name != 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + SHA: ${{ steps.generate.outputs.sha }} + run: | + if [ -z "$(git status --porcelain src/content/reference/python-api.mdx)" ]; then + echo "reference already matches lightpanda-python $SHA" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B python-reference + git add src/content/reference/python-api.mdx + git commit -m "Update the Python SDK API reference to lightpanda-python $SHA" + git push --force origin python-reference + if [ -z "$(gh pr list --head python-reference --state open --json number -q '.[].number')" ]; then + gh pr create --base main --head python-reference \ + --title "Update the Python SDK API reference to lightpanda-python $SHA" \ + --body "Regenerated \`src/content/reference/python-api.mdx\` from lightpanda-python $SHA. Served at https://lightpanda.io/docs/reference/python-api after the website's submodule bump." + fi diff --git a/redirects.mjs b/redirects.mjs index 6dfdfc8..0594d4e 100644 --- a/redirects.mjs +++ b/redirects.mjs @@ -10,6 +10,7 @@ export const basePath = '/docs' export const redirects = { + '/python': '/reference/python-api', '/quickstart/installation-and-setup': '/quickstart', '/quickstart/your-first-test': '/quickstart', '/quickstart/build-your-first-extraction-script': '/quickstart', diff --git a/scripts/generate-python-reference.py b/scripts/generate-python-reference.py new file mode 100644 index 0000000..7554688 --- /dev/null +++ b/scripts/generate-python-reference.py @@ -0,0 +1,419 @@ +"""Generate the Python SDK API reference page from the lightpanda package. + +The hand-written src/content/reference/python.mdx explains how the package fits +together; this script writes the exhaustive companion page, +src/content/reference/python-api.mdx, by walking the installed `lightpanda` +package with pdoc's Python API and emitting one MDX section per public class, +method, property, function and exception, with the signatures and docstrings +shipped in the code. Emitting MDX instead of pdoc's own HTML keeps the page +inside the Nextra site: sidebar, search, dark mode and deep links all work as +on any other page. + +Run it with the package installed from its main branch (that is what the +python-reference workflow does daily): + + uv run --no-project --python 3.13 --with 'pdoc==16.0.0' \\ + --with 'git+https://github.com/lightpanda-io/lightpanda-python' \\ + python scripts/generate-python-reference.py + +The output is deterministic for a given package, Python and pdoc version, so +the workflow can detect changes with `git status`. +""" + +from __future__ import annotations + +import argparse +import inspect +import re +import sys +from pathlib import Path + +import pdoc.doc +import pdoc.docstrings + +import lightpanda + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_OUT = ROOT / "src" / "content" / "reference" / "python-api.mdx" + +FRONTMATTER = """--- +title: Python API +description: Generated reference of every public class, method, property and exception in the lightpanda Python package, with the signatures and docstrings shipped in the code. +--- +""" + +BANNER = ( + "{/* Generated by scripts/generate-python-reference.py from lightpanda-python main. " + "Do not edit; rerun the script or wait for the python-reference workflow. */}" +) + +INTRO = ( + "Every public class, method, property and exception of the " + "[`lightpanda` package](https://pypi.org/project/lightpanda/), with the signatures and " + "docstrings shipped in the code. See [Python SDK](/reference/python) for a curated " + "overview of the same API and [Use the Python SDK](/guides/use-python) for practical " + "documentation. Every sync class has an asyncio twin with the same methods, " + "awaitable; the async sections below list only what the twin adds." +) + +FENCE_RE = re.compile(r"^\s*```") +CODE_SPAN_RE = re.compile(r"(`+)(.+?)\1", re.DOTALL) +MODULE_PREFIX_RE = re.compile(r"\blightpanda\.\w+\.") +LINK_RE = re.compile(r"\]\(#([a-z0-9-]+)\)") + + +def slug(*parts: str) -> str: + return "-".join(p.lower().replace("_", "-") for p in parts) + + +def clean_signature(text: str) -> str: + text = MODULE_PREFIX_RE.sub("", text) + text = text.replace("typing.", "") + return text + + +class Page: + def __init__(self) -> None: + self.lines: list[str] = [] + self.ids: set[str] = set() + + def heading(self, level: int, text: str, anchor: str) -> None: + if anchor in self.ids: + sys.exit(f"duplicate heading id: {anchor}") + self.ids.add(anchor) + self.lines.append(f"{'#' * level} {text} [#{anchor}]") + self.lines.append("") + + def para(self, text: str) -> None: + if text.strip(): + self.lines.append(text.rstrip()) + self.lines.append("") + + def fence(self, code: str, lang: str = "python") -> None: + self.lines.append(f"```{lang}") + self.lines.append(code.rstrip()) + self.lines.append("```") + self.lines.append("") + + def text(self) -> str: + text = "\n".join(self.lines).rstrip() + "\n" + missing = sorted(set(LINK_RE.findall(text)) - self.ids) + if missing: + sys.exit(f"links to missing heading ids: {', '.join(missing)}") + return text + + +# --- docstrings ------------------------------------------------------------- + + +def fence_indented_blocks(text: str) -> str: + """Wrap runs of indented lines (outside fences) in a text fence. + + Some docstrings carry hand-aligned tables, such as the value shapes of + `extract`; markdown would fold them into the preceding paragraph. + """ + out: list[str] = [] + block: list[str] = [] + in_fence = False + + def flush() -> None: + if block: + indent = min(len(line) - len(line.lstrip()) for line in block) + if out and out[-1].strip(): + out.append("") + out.append("```text") + out.extend(line[indent:] for line in block) + out.append("```") + out.append("") + block.clear() + + for line in text.split("\n"): + if FENCE_RE.match(line): + flush() + in_fence = not in_fence + out.append(line) + elif not in_fence and line.startswith(" ") and line.strip(): + block.append(line.rstrip()) + else: + flush() + out.append(line) + flush() + return "\n".join(out) + + +def escape_prose(text: str) -> str: + return text.replace("<", "<").replace("{", "\\{").replace("}", "\\}") + + +def render_prose(text: str, links: dict[str, str]) -> str: + """Escape MDX-significant characters outside inline code, link known symbols.""" + out: list[str] = [] + pos = 0 + for match in CODE_SPAN_RE.finditer(text): + out.append(escape_prose(text[pos : match.start()])) + span = match.group(0) + target = links.get(match.group(2).removesuffix("()")) + out.append(f"[{span}](#{target})" if target else span) + pos = match.end() + out.append(escape_prose(text[pos:])) + return "".join(out) + + +def render_docstring(doc: pdoc.doc.Doc, links: dict[str, str]) -> str: + raw = doc.docstring + if not raw.strip(): + return "" + text = pdoc.docstrings.convert(raw, "restructuredtext", doc.source_file) + text = fence_indented_blocks(text) + out: list[str] = [] + prose: list[str] = [] + in_fence = False + + def flush() -> None: + if prose: + out.append(render_prose("\n".join(prose), links)) + prose.clear() + + for line in text.split("\n"): + if FENCE_RE.match(line): + flush() + in_fence = not in_fence + out.append(line) + elif in_fence: + out.append(line) + else: + prose.append(line) + flush() + return "\n".join(out).strip() + + +# --- members --------------------------------------------------------------- + + +def is_public(member: pdoc.doc.Doc) -> bool: + if member.name.startswith("_"): + return False + if isinstance(member, pdoc.doc.Function) and member.obj.__name__ != member.name: + return False # alias of another method + return True + + +def public_members(cls: pdoc.doc.Class) -> list[pdoc.doc.Doc]: + return [m for m in cls.members.values() if is_public(m)] + + +def param_names(func: pdoc.doc.Function) -> list[str]: + return list(func.signature.parameters) + + +def twin_of(module: pdoc.doc.Module, cls: pdoc.doc.Class) -> pdoc.doc.Class | None: + """The sync class an `Async*` class mirrors, when both are public.""" + name = cls.name.removeprefix("Async") + twin = module.members.get(name) if name != cls.name else None + return twin if isinstance(twin, pdoc.doc.Class) else None + + +def listed_members(module: pdoc.doc.Module, cls: pdoc.doc.Class) -> list[pdoc.doc.Doc]: + """The members that get their own section: all of them, or for an async + twin only those the sync class lacks or spells with different parameters.""" + members = public_members(cls) + twin = twin_of(module, cls) + if twin is None: + return members + twin_members = {m.name: m for m in public_members(twin)} + + def added(member: pdoc.doc.Doc) -> bool: + other = twin_members.get(member.name) + if other is None: + return True + if isinstance(member, pdoc.doc.Function) and isinstance(other, pdoc.doc.Function): + return param_names(member) != param_names(other) + return False + + return [m for m in members if added(m)] + + +def member_anchor(module: pdoc.doc.Module, cls: pdoc.doc.Class, name: str) -> str | None: + """Heading id for `cls.name`, falling back to the sync twin's section.""" + if any(m.name == name for m in listed_members(module, cls)): + return slug(cls.name, name) + twin = twin_of(module, cls) + if twin is not None: + return member_anchor(module, twin, name) + return None + + +def function_code(func: pdoc.doc.Function, name: str | None = None) -> str: + decorators = "".join(f"{d}\n" for d in func.decorators) + signature = clean_signature(str(func.signature)) + return f"{decorators}{func.funcdef} {name or func.name}{signature}" + + +def variable_annotation(cls: pdoc.doc.Class, var: pdoc.doc.Variable) -> str: + annotation = var.annotation_str.removeprefix(":").strip() + if not annotation: + attr = getattr(cls.obj, var.name, None) + if isinstance(attr, property) and attr.fget is not None: + returns = inspect.signature(attr.fget).return_annotation + if returns is not inspect.Signature.empty: + annotation = inspect.formatannotation(returns) + return clean_signature(annotation) + + +def is_property(cls: pdoc.doc.Class, var: pdoc.doc.Variable) -> bool: + return isinstance(getattr(cls.obj, var.name, None), property) + + +def emit_member(page: Page, cls: pdoc.doc.Class, member: pdoc.doc.Doc, links: dict[str, str]) -> None: + page.heading(3, f"{cls.name}.{member.name}", slug(cls.name, member.name)) + doc = render_docstring(member, links) + if isinstance(member, pdoc.doc.Function): + page.fence(function_code(member)) + page.para(doc) + elif isinstance(member, pdoc.doc.Variable): + annotation = variable_annotation(cls, member) + page.fence(f"{member.name}: {annotation}" if annotation else member.name) + if is_property(cls, member): + page.para(f"*Property.* {doc}".rstrip()) + else: + page.para(doc) + + +def class_code(cls: pdoc.doc.Class) -> str: + init = cls.members.get("__init__") + if isinstance(init, pdoc.doc.Function) and "__init__" in vars(cls.obj): + return f"class {cls.name}{clean_signature(str(init.signature_without_self))}" + return f"class {cls.name}" + + +def emit_class(page: Page, module: pdoc.doc.Module, cls: pdoc.doc.Class, links: dict[str, str]) -> None: + page.heading(2, cls.name, slug(cls.name)) + page.para(render_docstring(cls, links)) + page.fence(class_code(cls)) + init = cls.members.get("__init__") + if isinstance(init, pdoc.doc.Function) and "__init__" in vars(cls.obj): + page.para(render_docstring(init, links)) + managers = [] + if "__enter__" in cls.members: + managers.append("`with`") + if "__aenter__" in cls.members: + managers.append("`async with`") + if managers: + page.para(f"Usable as a context manager ({' and '.join(managers)}).") + + members = listed_members(module, cls) + twin = twin_of(module, cls) + if twin is not None: + page.para( + f"Every public method and property of [`{twin.name}`](#{slug(twin.name)}) exists on " + f"`{cls.name}` with the same name and signature; methods are coroutines to `await`. " + + ( + f"Only the members `{cls.name}` adds are listed below." + if members + else f"`{cls.name}` adds no members of its own." + ) + ) + for member in members: + emit_member(page, cls, member, links) + + +def emit_function(page: Page, func: pdoc.doc.Function, links: dict[str, str]) -> None: + page.heading(2, func.name, slug(func.name)) + page.fence(function_code(func)) + page.para(render_docstring(func, links)) + + +def emit_exception(page: Page, cls: pdoc.doc.Class, links: dict[str, str]) -> None: + page.heading(3, cls.name, slug(cls.name)) + bases = [name for _, name, qualname in cls.bases if qualname != "builtins.object"] + code = f"class {cls.name}({', '.join(bases)})" if bases else f"class {cls.name}" + init = cls.members.get("__init__") + if isinstance(init, pdoc.doc.Function) and "__init__" in vars(cls.obj): + code += f"\n{cls.name}{clean_signature(str(init.signature_without_self))}" + page.fence(code) + page.para(render_docstring(cls, links)) + attrs = [m for m in cls.own_members if isinstance(m, pdoc.doc.Variable) and is_public(m)] + if attrs: + for attr in attrs: + annotation = variable_annotation(cls, attr) + line = f"- `{attr.name}`" + (f": `{annotation}`" if annotation else "") + doc = render_docstring(attr, links) + if doc: + line += f". {doc}" + page.lines.append(line) + page.lines.append("") + + +# --- page ------------------------------------------------------------------ + + +def collect_links(module: pdoc.doc.Module, names: list[str]) -> dict[str, str]: + """Map symbol spellings found in docstrings to heading ids.""" + links: dict[str, str] = {} + for name in names: + doc = module.members[name] + links[name] = slug(name) + if isinstance(doc, pdoc.doc.Class): + for member in public_members(doc): + anchor = member_anchor(module, doc, member.name) + if anchor: + links[f"{name}.{member.name}"] = anchor + return links + + +def class_links( + module: pdoc.doc.Module, links: dict[str, str], cls: pdoc.doc.Class +) -> dict[str, str]: + """Resolve bare member names (`start()`) against the class being rendered.""" + scoped = dict(links) + for member in public_members(cls): + anchor = member_anchor(module, cls, member.name) + if anchor: + scoped.setdefault(member.name, anchor) + return scoped + + +def generate() -> str: + module = pdoc.doc.Module(lightpanda) + names = [n for n in lightpanda.__all__ if n in module.members] + links = collect_links(module, names) + page = Page() + + page.lines.append(FRONTMATTER.rstrip()) + page.lines.append(BANNER) + page.lines.append("") + page.lines.append("# Python API") + page.lines.append("") + page.para(INTRO) + page.para(render_docstring(module, links)) + + exceptions: list[pdoc.doc.Class] = [] + for name in names: + doc = module.members[name] + if isinstance(doc, pdoc.doc.Class): + if issubclass(doc.obj, BaseException): + exceptions.append(doc) + else: + emit_class(page, module, doc, class_links(module, links, doc)) + elif isinstance(doc, pdoc.doc.Function): + emit_function(page, doc, links) + + if exceptions: + page.heading(2, "Exceptions", "exceptions") + for doc in exceptions: + emit_exception(page, doc, class_links(module, links, doc)) + + return page.text() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT, help="output .mdx path") + args = parser.parse_args() + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(generate(), encoding="utf-8", newline="\n") + print(f"wrote {args.out.relative_to(ROOT) if args.out.is_relative_to(ROOT) else args.out}") + + +if __name__ == "__main__": + main() diff --git a/src/content/guides/use-python.mdx b/src/content/guides/use-python.mdx index 3c15014..b48af54 100644 --- a/src/content/guides/use-python.mdx +++ b/src/content/guides/use-python.mdx @@ -164,7 +164,7 @@ asyncio.run(main()) Every browser action is a `Session` method, typed and documented in your IDE, with the action and its arguments in snake_case (`wait_for_selector`, `backend_node_id`). -Find every method's arguments in the [Python SDK reference](/reference/python), or browse the generated API reference at [lightpanda.io/lightpanda-python](https://lightpanda.io/lightpanda-python/). +Find every method's arguments in the [Python SDK reference](/reference/python), or browse the generated [Python API](/reference/python-api) reference. ## Replay a saved script diff --git a/src/content/reference/_meta.ts b/src/content/reference/_meta.ts index ea6c7c1..7f6d572 100644 --- a/src/content/reference/_meta.ts +++ b/src/content/reference/_meta.ts @@ -11,6 +11,7 @@ const meta: MetaRecord = { 'mcp-tools': 'MCP tools', pandascript: 'PandaScript', python: 'Python SDK', + 'python-api': 'Python API', } export default meta diff --git a/src/content/reference/python-api.mdx b/src/content/reference/python-api.mdx new file mode 100644 index 0000000..cb3ca3f --- /dev/null +++ b/src/content/reference/python-api.mdx @@ -0,0 +1,848 @@ +--- +title: Python API +description: Generated reference of every public class, method, property and exception in the lightpanda Python package, with the signatures and docstrings shipped in the code. +--- +{/* Generated by scripts/generate-python-reference.py from lightpanda-python main. Do not edit; rerun the script or wait for the python-reference workflow. */} + +# Python API + +Every public class, method, property and exception of the [`lightpanda` package](https://pypi.org/project/lightpanda/), with the signatures and docstrings shipped in the code. See [Python SDK](/reference/python) for a curated overview of the same API and [Use the Python SDK](/guides/use-python) for practical documentation. Every sync class has an asyncio twin with the same methods, awaitable; the async sections below list only what the twin adds. + +Lightpanda for Python: a lightweight headless browser. + +```python +from lightpanda import Browser + +with Browser() as b: + page = b.new_session() + page.goto(url="https://example.com") + data = page.extract(schema={"title": "h1"}) +``` + +The same API is available for asyncio: + +```python +from lightpanda import AsyncBrowser + +async with AsyncBrowser() as b: + page = await b.new_session() + await page.goto(url="https://example.com") + data = await page.extract(schema={"title": "h1"}) +``` + +For Playwright or Puppeteer code, [``CDPServer``](#cdpserver) runs the browser's own +Chrome DevTools Protocol server and hands you the endpoint to connect to +(see its docs for an example). For Selenium, [``BiDiServer``](#bidiserver) serves WebDriver +BiDi the same way and hands you the ``command_executor`` URL. + +## Browser [#browser] + +A lightpanda browser process. Spawns the bundled binary on first use. + +Not fork-inheritable: after ``os.fork()``/``multiprocessing``, create a +fresh Browser in the child. + +```python +class Browser( + binary: str | os.PathLike | None = None, + env: dict[str, str] | None = None, + timeout: float = 300.0, + verbose: bool = False, + args: Sequence[str] = () +) +``` + +``args`` are extra CLI flags for the spawned browser process +(e.g. ``["--http-cache-dir", path]`` or cookie flags). + +Usable as a context manager (`with`). + +### Browser.tools [#browser-tools] + +```python +tools: dict[str, dict] +``` + +*Property.* Tool name → \{description, schema\}, as reported by the browser. + +### Browser.new_session [#browser-new-session] + +```python +def new_session(self) -> Session +``` + +### Browser.close [#browser-close] + +```python +def close(self) -> None +``` + +## Session [#session] + +One isolated browsing context (own page, cookies, memory). + +Do not construct directly — use [`Browser.new_session()`](#browser-new-session). + +```python +class Session(browser: Browser, session_id: str) +``` + +Usable as a context manager (`with`). + +### Session.id [#session-id] + +```python +id: str +``` + +*Property.* + +### Session.call [#session-call] + +```python +def call(self, tool: str, **kwargs) +``` + +Invoke a browser tool by name. The generated methods route here. + +Returns parsed JSON for JSON-carrying tools, ``bytes`` for image +results ([``screenshot``](#session-screenshot) without ``path``), otherwise the result text. + +### Session.close [#session-close] + +```python +def close(self) -> None +``` + +### Session.click [#session-click] + +```python +def click( + self, + *, + selector: str | None = None, + backend_node_id: int | None = None +) -> Any +``` + +Click on an interactive element. Provide either a CSS selector (preferred for reproducibility) or a backendNodeId. Returns the current page URL and title after the click. + +### Session.console_logs [#session-console-logs] + +```python +def console_logs(self) -> Any +``` + +Get buffered console.log/warn/error messages from the current page. Returns all messages since last call and clears the buffer. + +### Session.detect_forms [#session-detect-forms] + +```python +def detect_forms(self, *, url: str | None = None, timeout: int | None = None) -> Any +``` + +Detect all forms on the page and return their structure including fields, types, and required status. If a url is provided, it navigates to that url first. + +### Session.evaluate [#session-evaluate] + +```python +def evaluate( + self, + *, + script: str, + url: str | None = None, + timeout: int | None = None, + save: str | None = None +) -> Any +``` + +Evaluate JavaScript in the current page context — an escape hatch for page-side logic the dedicated tools can't express; prefer [`extract`](#session-extract) for data and click/fill/etc. for actions. It runs in the page, so it cannot see the agent script's variables or builtins — interpolate any value into the `script` string. A bare trailing expression yields its value; top-level `await` and `return` are supported (the body then runs as an async function, so use `return` to produce a value). Objects and arrays return as JSON, so no `JSON.stringify` is needed. If a url is provided, it navigates there first. The `globalThis.lp` object exposes a Session-scoped bridge store: values written via `lp.foo = ...` auto-sync at end of evaluate, surviving navigation; values previously set via `/extract save=` or `/evaluate save=` appear as `lp.`. + +### Session.extract [#session-extract] + +```python +def extract(self, *, schema: str | dict | list, save: str | None = None) -> Any +``` + +Extract structured data from the current page (navigate first). `schema` is a JSON object (passed as a string) mapping output field names to CSS-selector specs. It is NOT a JSON Schema — no "type"/"properties" wrappers; the keys ARE your output fields. Value shapes: + +```text +"" → first match's text (trimmed; null if no match) +[""] → every match's text (string[]) +{"selector":"","attr":""} → first match's attribute value (href/src resolved to absolute URLs) +[{"selector":"","attr":""}] → every match's attribute (string[]) +[{"selector":"","fields":{…}}] → one object per match; field selectors resolve relative to that match and accept any shape above ("" = the match's own text; nest arrays for per-item sub-lists) +``` + +Add "limit": N inside any array's object spec to cap matches. +Every extracted value is a string or null — parse numbers downstream. An empty array is a valid result, but if ALL top-level keys miss, the call errors: inspect the page (tree/markdown) and retry with corrected selectors. +Finish data tasks with extract — it is the only read recorded as a replayable `extract(...)` script call; answers lifted from [`markdown`](#session-markdown) text in chat are not. + +Examples (schema → result): + +```text +{"karma": "#karma"} → {"karma":"42"} +{"items": [".story .title"]} → {"items":["Title 1","Title 2"]} +{"top3": [{"selector":".story .title","limit":3}]} → {"top3":["A","B","C"]} +{"links": [{"selector":"a.title","attr":"href"}]} → {"links":["https://site/a","https://site/b"]} +{"stories": [{"selector":".athing","fields":{"title":".titleline","rank":".rank"}}]} → {"stories":[{"title":"Foo","rank":"1"}]} +``` + +### Session.fill [#session-fill] + +```python +def fill( + self, + *, + value: str, + selector: str | None = None, + backend_node_id: int | None = None +) -> Any +``` + +Fill text into an input element. Provide either a CSS selector (preferred for reproducibility) or a backendNodeId. + +### Session.find_element [#session-find-element] + +```python +def find_element(self, *, role: str | None = None, name: str | None = None) -> Any +``` + +Find interactive elements by role and/or accessible name. Returns matching elements with their backend node IDs. Useful for locating specific elements without parsing the full semantic tree. + +### Session.get_cookies [#session-get-cookies] + +```python +def get_cookies(self, *, url: str | None = None, all: bool | None = None) -> Any +``` + +Cookies stored in the browser. Defaults to cookies whose domain matches the current page's host. Pass `url=` to filter for another host, or `all=true` to dump every cookie regardless of host. Useful for debugging authentication and session state. + +### Session.get_env [#session-get-env] + +```python +def get_env(self, *, name: str | None = None) -> Any +``` + +With `name`: read an LP_* env var (other namespaces report as not set) — for non-secret config only (base URLs, flags). Without `name`: list LP_* names that are set (no values) — safe credential discovery. For secrets, pass `$LP_*` placeholders in tool args; never request a credential by name (the value would land in your context). + +### Session.get_url [#session-get-url] + +```python +def get_url(self) -> Any +``` + +Current page URL. The browser may already have a page loaded (command, replayed script) not visible in this conversation — call this before assuming nothing is loaded when the user references the current page/site. Also useful to verify a navigation or detect a redirect. + +### Session.goto [#session-goto] + +```python +def goto( + self, + *, + url: str, + timeout: int | None = None, + wait_until: str | None = None +) -> Any +``` + +Navigate to a specified URL and load the page in memory so it can be reused later for info extraction. + +### Session.hover [#session-hover] + +```python +def hover( + self, + *, + selector: str | None = None, + backend_node_id: int | None = None +) -> Any +``` + +Hover over an element, triggering mouseover and mouseenter events. Provide either a CSS selector (preferred for reproducibility) or a backendNodeId. Useful for menus, tooltips, and hover states. + +### Session.html [#session-html] + +```python +def html( + self, + *, + selector: str | None = None, + backend_node_id: int | None = None, + max_bytes: int | None = None, + strip: dict | None = None, + url: str | None = None, + timeout: int | None = None +) -> Any +``` + +Raw HTML for the document or, with `selector`/`backendNodeId`, a single node's outerHTML. Verbose; use only when you need attributes that markdown discards. + +### Session.interactive_elements [#session-interactive-elements] + +```python +def interactive_elements(self, *, url: str | None = None, timeout: int | None = None) -> Any +``` + +Extract interactive elements from the opened page. If a url is provided, it navigates to that url first. + +### Session.links [#session-links] + +```python +def links( + self, + *, + limit: int | None = None, + url: str | None = None, + timeout: int | None = None +) -> Any +``` + +Extract the visible links in the opened page as JSON objects with `text` (anchor text, falling back to aria-label/title/image alt), `href` (resolved URL), and `backendNodeId` (pass to click/nodeDetails). One entry per href; hidden links are omitted. If a url is provided, it navigates to that url first. + +### Session.markdown [#session-markdown] + +```python +def markdown( + self, + *, + selector: str | None = None, + backend_node_id: int | None = None, + max_bytes: int | None = None, + url: str | None = None, + timeout: int | None = None +) -> Any +``` + +Render the page (or a subtree) as markdown. Scope with `selector` or `backendNodeId` to read just the relevant region — full-page markdown is the last resort. Use `maxBytes` to cap long pages. + +### Session.node_details [#session-node-details] + +```python +def node_details(self, *, backend_node_id: int) -> Any +``` + +Details for a node by backendNodeId: a ready-to-use CSS `selector` that resolves to the node (the first match, as click/fill resolve it), plus tag, role, name, interactivity, disabled, value, input type, placeholder, href, id, class, checked, select options. The canonical way to turn a tree backendNodeId into a CSS selector. + +### Session.press [#session-press] + +```python +def press( + self, + *, + key: str, + selector: str | None = None, + backend_node_id: int | None = None +) -> Any +``` + +Press a keyboard key, dispatching keydown and keyup events. Use key names like 'Enter', 'Tab', 'Escape', 'ArrowDown', 'Backspace', or single characters like 'a', '1'. Common shorthand is normalized: 'enter'/'return' → 'Enter', 'esc' → 'Escape', 'up'/'down'/'left'/'right' → 'Arrow*', 'space' → ' '. Pressing 'Enter' on a form input or submit button triggers implicit form submission. + +### Session.screenshot [#session-screenshot] + +```python +def screenshot( + self, + *, + path: str | None = None, + selector: str | None = None, + backend_node_id: int | None = None, + full_page: bool | None = None, + url: str | None = None, + timeout: int | None = None +) -> Any +``` + +Render the page, or one node, as a PNG: the text layout Lightpanda computes, not a pixel-accurate browser rendering (no images, fonts or CSS colours). With `path`, writes the file at full size and returns its location; without it, returns the image inline where the client can display one, at most 1280px wide and 4096px tall. Use it to see spatial layout; read content with [`markdown`](#session-markdown)/[`tree`](#session-tree). + +### Session.scroll [#session-scroll] + +```python +def scroll( + self, + *, + backend_node_id: int | None = None, + x: int | None = None, + y: int | None = None +) -> Any +``` + +Scroll the page or a specific element. Returns the scroll position and current page URL and title. + +### Session.search [#session-search] + +```python +def search(self, *, query: str, timeout: int | None = None) -> Any +``` + +Run a web search and return results as markdown: a numbered list of \{title, url, snippet\}. Search tries brave, tavily, exa, then keenable in order, each when its API key (BRAVE_API_KEY, TAVILY_API_KEY, EXA_API_KEY or KEENABLE_API_KEY) is set; keenable also works without a key through its public endpoint (rate-limited per client IP). Prefer this over goto-ing google.com/search directly (Google blocks the browser on User-Agent/TLS). The browser does not navigate — to open a result, use [`goto`](#session-goto) with its URL. + +### Session.select_option [#session-select-option] + +```python +def select_option( + self, + *, + value: str, + selector: str | None = None, + backend_node_id: int | None = None +) -> Any +``` + +Select an option in a <select> dropdown element by its value. Provide either a CSS selector (preferred for reproducibility) or a backendNodeId. Dispatches input and change events. + +### Session.set_checked [#session-set-checked] + +```python +def set_checked( + self, + *, + checked: bool, + selector: str | None = None, + backend_node_id: int | None = None +) -> Any +``` + +Check or uncheck a checkbox or radio button. Provide either a CSS selector (preferred for reproducibility) or a backendNodeId. Dispatches input, change, and click events. + +### Session.structured_data [#session-structured-data] + +```python +def structured_data(self, *, url: str | None = None, timeout: int | None = None) -> Any +``` + +Extract structured data (like JSON-LD, OpenGraph, etc) from the opened page. If a url is provided, it navigates to that url first. + +### Session.tree [#session-tree] + +```python +def tree( + self, + *, + url: str | None = None, + timeout: int | None = None, + backend_node_id: int | None = None, + max_depth: int | None = None +) -> Any +``` + +Simplified semantic DOM tree (role, name, value, backendNodeId per node). Pass `backendNodeId` to scope, `maxDepth` to limit depth. + +### Session.wait_for_script [#session-wait-for-script] + +```python +def wait_for_script(self, *, script: str, timeout: int | None = None) -> Any +``` + +Wait until a JS expression returns truthy. Re-evaluates on each tick of the event loop. Use for synchronization beyond what CSS selectors can express — e.g. `window.dataLoaded === true`, `document.readyState === 'complete'`, `document.querySelectorAll('.row').length >= 5`. + +### Session.wait_for_selector [#session-wait-for-selector] + +```python +def wait_for_selector(self, *, selector: str, timeout: int | None = None) -> Any +``` + +Wait for an element matching a CSS selector to appear in the page. Returns the backend node ID of the matched element. + +### Session.wait_for_state [#session-wait-for-state] + +```python +def wait_for_state(self, *, state: str, timeout: int | None = None) -> Any +``` + +Wait for the CURRENT page to reach a load state (no navigation). After a [`goto`](#session-goto), the page is returned at the fast `load` snapshot, so content rendered by post-load JS (XHR-loaded lists, feeds, search results) may still be missing. When a read looks incomplete — empty lists, spinners, skeletons — call this with 'networkidle' and re-read. Prefer 'networkidle'; 'done' can be slow on sites with constant background activity (ads, polling). + +## run_script [#run-script] + +```python +def run_script( + script: str | os.PathLike, + env: dict[str, str] | None = None, + binary: str | os.PathLike | None = None, + timeout: float | None = None +) -> str +``` + +Replay a saved lightpanda script (no LLM) and return its stdout. + +``env`` entries (e.g. ``LP_*`` placeholder values) are added to the +child's environment. Raises [`ScriptError`](#scripterror) on a non-zero exit. + +## AsyncBrowser [#asyncbrowser] + +A lightpanda browser process, driven from asyncio. + +The subprocess is spawned by [`start()`](#asyncbrowser-start) — called automatically on +``async with`` entry and by [`new_session()`](#browser-new-session). Not fork-inheritable, +same as [`Browser`](#browser). + +```python +class AsyncBrowser( + binary: str | os.PathLike | None = None, + env: dict[str, str] | None = None, + timeout: float = 300.0, + verbose: bool = False, + args: Sequence[str] = (), + max_concurrency: int = 32 +) +``` + +``binary``/``env``/``timeout``/``verbose``/``args`` are forwarded +to [`Browser`](#browser). ``max_concurrency`` caps concurrently executing +tool calls across this browser's sessions (worker threads are +created lazily). + +Usable as a context manager (`async with`). + +Every public method and property of [`Browser`](#browser) exists on `AsyncBrowser` with the same name and signature; methods are coroutines to `await`. Only the members `AsyncBrowser` adds are listed below. + +### AsyncBrowser.wrap [#asyncbrowser-wrap] + +```python +@classmethod +def wrap( + cls, + browser: Browser, + max_concurrency: int = 32 +) -> AsyncBrowser +``` + +Adopt an already-running [`Browser`](#browser) — the migration path for +driving existing sync setup from asyncio. [`close()`](#browser-close) shuts down +the facade but leaves the wrapped browser running. + +### AsyncBrowser.start [#asyncbrowser-start] + +```python +async def start(self) -> AsyncBrowser +``` + +Spawn the browser process and fetch its tool list. Idempotent. + +### AsyncBrowser.session [#asyncbrowser-session] + +```python +@contextlib.asynccontextmanager +async def session(self) +``` + +``async with browser.session() as page:`` — a session scoped to +the block and closed on exit. Use [`new_session()`](#browser-new-session) for the +unscoped form. + +## AsyncSession [#asyncsession] + +One isolated browsing context (own page, cookies, memory), async. + +Do not construct directly — use [`AsyncBrowser.new_session()`](#browser-new-session). + +```python +class AsyncSession( + session: Session, + executor: concurrent.futures.thread.ThreadPoolExecutor +) +``` + +Usable as a context manager (`async with`). + +Every public method and property of [`Session`](#session) exists on `AsyncSession` with the same name and signature; methods are coroutines to `await`. `AsyncSession` adds no members of its own. + +## run_script_async [#run-script-async] + +```python +async def run_script_async( + script: str | os.PathLike, + env: dict[str, str] | None = None, + binary: str | os.PathLike | None = None, + timeout: float | None = None +) -> str +``` + +Async variant of `lightpanda.run_script()` (runs in a worker thread). + +## CDPServer [#cdpserver] + +A lightpanda process serving the Chrome DevTools Protocol on 127.0.0.1. + +```python +from lightpanda import CDPServer +from playwright.sync_api import sync_playwright + +with CDPServer() as server, sync_playwright() as p: + browser = p.chromium.connect_over_cdp(server.ws_endpoint) + page = browser.new_context().new_page() + page.goto("https://example.com") +``` + +Every connected client gets its own browser; up to 16 connect at once +by default (``args=["--cdp-max-connections", "N"]`` to change). The +process is stopped by [`close()`](#cdpserver-close) / leaving the ``with`` block, and on +Linux also when the interpreter dies. + +```python +class CDPServer( + binary: str | os.PathLike | None = None, + env: dict[str, str] | None = None, + verbose: bool = False, + args: Sequence[str] = (), + port: int | None = None +) +``` + +[``port``](#cdpserver-port) pins the listening port (default: a free one). ``args`` +are extra ``lightpanda serve`` flags; pass ``port=`` rather than +``--port``. ``verbose`` lets the browser log through to stderr. + +Usable as a context manager (`with`). + +### CDPServer.ws_endpoint [#cdpserver-ws-endpoint] + +```python +ws_endpoint: str +``` + +*Property.* The CDP WebSocket URL, ``ws://127.0.0.1:/``. + +Keep it as is: the server only upgrades on path ``/`` and only +accepts an IP-literal or ``localhost`` host. + +### CDPServer.version [#cdpserver-version] + +```python +def version(self) -> dict +``` + +The ``/json/version`` document (browser, protocol version, +``webSocketDebuggerUrl``). + +### CDPServer.port [#cdpserver-port] + +```python +port: int +``` + +*Property.* + +### CDPServer.http_endpoint [#cdpserver-http-endpoint] + +```python +http_endpoint: str +``` + +*Property.* ``http://127.0.0.1:``, the server's HTTP root: what Puppeteer +(``browserURL``) and Playwright (``connect_over_cdp`` with an http URL) +discover the CDP WebSocket from, and Selenium's ``command_executor``. + +### CDPServer.close [#cdpserver-close] + +```python +def close(self) -> None +``` + +## AsyncCDPServer [#asynccdpserver] + +[`CDPServer`](#cdpserver) for asyncio: the process is spawned by +[`start()`](#asynccdpserver-start), called automatically on ``async with`` entry. + +```python +async with AsyncCDPServer() as server, async_playwright() as p: + browser = await p.chromium.connect_over_cdp(server.ws_endpoint) +``` + +```python +class AsyncCDPServer( + binary: str | os.PathLike | None = None, + env: dict[str, str] | None = None, + verbose: bool = False, + args: Sequence[str] = (), + port: int | None = None +) +``` + +Arguments are forwarded to the sync class. + +Usable as a context manager (`async with`). + +Every public method and property of [`CDPServer`](#cdpserver) exists on `AsyncCDPServer` with the same name and signature; methods are coroutines to `await`. Only the members `AsyncCDPServer` adds are listed below. + +### AsyncCDPServer.start [#asynccdpserver-start] + +```python +async def start(self) +``` + +Spawn the server process. Idempotent. + +## BiDiServer [#bidiserver] + +A lightpanda process serving WebDriver BiDi on 127.0.0.1. + +```python +from lightpanda import BiDiServer +from selenium import webdriver +from selenium.webdriver.common.options import ArgOptions + +options = ArgOptions() +options.web_socket_url = True # ask for a WebDriver BiDi session + +with BiDiServer() as server: + driver = webdriver.Remote(command_executor=server.http_endpoint, options=options) + context = driver.browsing_context.create(type="tab") + driver.browsing_context.navigate(context=context, url="https://example.com", wait="complete") + print(driver.script.execute("() => document.title", context_id=context)["value"]) + driver.quit() +``` + +[`http_endpoint`](#bidiserver-http-endpoint) is Selenium's ``command_executor``. The browser +serves the BiDi modules (``session``, ``browser``, ``browsingContext``, +``script``, ``input``) over the WebSocket plus the classic session +bootstrap (``GET /status``, ``POST /session`` with the ``webSocketUrl`` +capability, ``DELETE /session/``); other classic WebDriver commands +such as Selenium's ``driver.get`` or ``find_element`` are not served, so +drive the page through ``driver.browsing_context`` and ``driver.script`` +with an explicit context, created first as above. Pass +``args=["--protocol", "cdp"]`` to serve CDP on the same port as well +(``--protocol`` is additive). The process is stopped by [`close()`](#bidiserver-close) / +leaving the ``with`` block, and on Linux also when the interpreter dies. + +```python +class BiDiServer( + binary: str | os.PathLike | None = None, + env: dict[str, str] | None = None, + verbose: bool = False, + args: Sequence[str] = (), + port: int | None = None +) +``` + +[``port``](#bidiserver-port) pins the listening port (default: a free one). ``args`` +are extra ``lightpanda serve`` flags; pass ``port=`` rather than +``--port``. ``verbose`` lets the browser log through to stderr. + +Usable as a context manager (`with`). + +### BiDiServer.bidi_endpoint [#bidiserver-bidi-endpoint] + +```python +bidi_endpoint: str +``` + +*Property.* The session-less BiDi WebSocket URL, ``ws://127.0.0.1:/session``, +for clients that speak BiDi directly (``session.new`` over the socket). +A session bootstrapped through ``POST /session`` gets its own socket at +``/``, returned as the ``webSocketUrl`` +capability. + +Keep the IP literal: the WebSocket upgrade rejects any ``Origin`` +header and only accepts an IP-literal or ``localhost`` host. + +### BiDiServer.status [#bidiserver-status] + +```python +def status(self) -> dict +``` + +The ``GET /status`` value, ``{"ready": True, "message": ""}``. + +### BiDiServer.port [#bidiserver-port] + +```python +port: int +``` + +*Property.* + +### BiDiServer.http_endpoint [#bidiserver-http-endpoint] + +```python +http_endpoint: str +``` + +*Property.* ``http://127.0.0.1:``, the server's HTTP root: what Puppeteer +(``browserURL``) and Playwright (``connect_over_cdp`` with an http URL) +discover the CDP WebSocket from, and Selenium's ``command_executor``. + +### BiDiServer.close [#bidiserver-close] + +```python +def close(self) -> None +``` + +## AsyncBiDiServer [#asyncbidiserver] + +[`BiDiServer`](#bidiserver) for asyncio: the process is spawned by +[`start()`](#asyncbidiserver-start), called automatically on ``async with`` entry. + +```python +class AsyncBiDiServer( + binary: str | os.PathLike | None = None, + env: dict[str, str] | None = None, + verbose: bool = False, + args: Sequence[str] = (), + port: int | None = None +) +``` + +Arguments are forwarded to the sync class. + +Usable as a context manager (`async with`). + +Every public method and property of [`BiDiServer`](#bidiserver) exists on `AsyncBiDiServer` with the same name and signature; methods are coroutines to `await`. Only the members `AsyncBiDiServer` adds are listed below. + +### AsyncBiDiServer.start [#asyncbidiserver-start] + +```python +async def start(self) +``` + +Spawn the server process. Idempotent. + +## Exceptions [#exceptions] + +### LightpandaError [#lightpandaerror] + +```python +class LightpandaError(Exception) +``` + +Base error for the lightpanda package. + +### ProcessError [#processerror] + +```python +class ProcessError(LightpandaError) +``` + +The browser binary could not be found, started, or reached. + +### ProtocolError [#protocolerror] + +```python +class ProtocolError(LightpandaError) +ProtocolError(message: str, code: int | None = None) +``` + +JSON-RPC level failure (invalid request, timeout, internal error). + +- `code` + +### ScriptError [#scripterror] + +```python +class ScriptError(LightpandaError) +ScriptError(message: str, returncode: int, stdout: str = '', stderr: str = '') +``` + +A script replay ([`run_script`](#run-script)) exited with a failure. + +- `returncode` +- `stdout` +- `stderr` + +### ToolError [#toolerror] + +```python +class ToolError(LightpandaError) +``` + +A browser tool reported failure (bad selector, JS exception, ...). diff --git a/src/content/reference/python.mdx b/src/content/reference/python.mdx index e0db810..c60d4b3 100644 --- a/src/content/reference/python.mdx +++ b/src/content/reference/python.mdx @@ -5,11 +5,11 @@ description: Reference of the classes and methods in the Lightpanda Python packa # Python SDK -The [`lightpanda` package](https://pypi.org/project/lightpanda/) exposes `Browser`/`AsyncBrowser`, which spawn and manage the bundled binary, and `Session`/`AsyncSession`, with one method per browser action. See [Use the Python SDK](/guides/use-python) for practical documentation. +The [`lightpanda` package](https://pypi.org/project/lightpanda/) exposes `Browser`/`AsyncBrowser`, which spawn and manage the bundled binary, and `Session`/`AsyncSession`, with one method per browser action. See [Use the Python SDK](/guides/use-python) for practical documentation, and the [Python API](/reference/python-api) page for every signature and docstring as shipped in the package. ## Browser -`Browser()` spawns the bundled binary when constructed. It is not fork-inheritable: create a fresh instance in a forked child. +[`Browser()`](/reference/python-api#browser) spawns the bundled binary when constructed. It is not fork-inheritable: create a fresh instance in a forked child. | Argument | Default | Description | |---|---|---| @@ -29,7 +29,7 @@ The [`lightpanda` package](https://pypi.org/project/lightpanda/) exposes `Browse ## AsyncBrowser -`AsyncBrowser` mirrors `Browser` for asyncio: every call runs on a browser-owned thread pool, so the event loop is never blocked. +[`AsyncBrowser`](/reference/python-api#asyncbrowser) mirrors `Browser` for asyncio: every call runs on a browser-owned thread pool, so the event loop is never blocked. | Argument | Default | Description | |---|---|---| @@ -49,7 +49,7 @@ The [`lightpanda` package](https://pypi.org/project/lightpanda/) exposes `Browse ## Session and AsyncSession -`Browser.new_session()` and `AsyncBrowser.new_session()` are the only way to obtain a `Session` or `AsyncSession`; do not construct one directly. +`Browser.new_session()` and `AsyncBrowser.new_session()` are the only way to obtain a [`Session`](/reference/python-api#session) or [`AsyncSession`](/reference/python-api#asyncsession); do not construct one directly. | Member | Description | |---|---| @@ -61,7 +61,7 @@ Sessions are context managers too: `with browser.new_session() as page:` closes ## Calling an action -Every browser action is a method on `Session`/`AsyncSession`, keyword-only, with the action and its arguments in snake_case: the `waitForSelector` action is `wait_for_selector`, and its `backendNodeId` argument is `backend_node_id`. The methods are generated from the bundled browser's action schemas, so the signatures and docstrings your IDE shows come straight from the binary. The generated reference for the latest release is published at [lightpanda.io/lightpanda-python](https://lightpanda.io/lightpanda-python/). +Every browser action is a method on `Session`/`AsyncSession`, keyword-only, with the action and its arguments in snake_case: the `waitForSelector` action is `wait_for_selector`, and its `backendNodeId` argument is `backend_node_id`. The methods are generated from the bundled browser's action schemas, so the signatures and docstrings your IDE shows come straight from the binary. The [Python API](/reference/python-api#session) page lists every method with its exact signature and docstring. A failed action raises `ToolError`. @@ -149,7 +149,7 @@ These methods block until the page reaches a condition: ## Script replay -`run_script` and `run_script_async` (its awaitable variant, run in a worker thread) replay a saved [PandaScript](/reference/pandascript) with no LLM call, by running `lightpanda run