MCP server, qc, plot, merge --common-rows, mcp watch; version 0.10.0

Cargo.toml

@@ -9,7 +9,7 @@ edition = "2021"
 readme = "README.md"
 # Own version (decoupled from workspace.version): data-beans evolves its CLI /
 # QC API independently of the shared utility crates.
-version = "0.9.1"
+version = "0.10.0"
 rust-version = "1.91"
 # Ship only the sources, README and license: nothing else in the working tree
 # (data stores, saved name lists, scratch files) can end up on crates.io.
@@ -27,7 +27,7 @@ path = "src/sim_main.rs"
 required-features = ["sim"]
 
 [features]
-default = ["aux", "tui", "tensor", "ndarray"]
+default = ["aux", "tui", "tensor", "ndarray", "mcp", "plot"]
 # Read sparse data into dense candle `Tensor`s (`read_*_tensor`). Off, no
 # candle is linked: for crates that only need matrices.
 tensor = ["legume-numeric/tensor"]
@@ -40,7 +40,13 @@ aux = []
 # Full-screen terminal views (ratatui): the CLI's interactive sessions, and
 # `data_beans::interactive` for downstream crates building their own. The
 # views run in legume-tui sessions, which a dropped connection leaves running.
-tui = ["dep:ratatui", "dep:legume-tui"]
+tui = ["dep:ratatui", "dep:legume-tui", "legume-mcp?/tui"]
+# `data-beans mcp`: the commands as Model Context Protocol tools for an
+# agent, over stdio.
+mcp = ["dep:legume-mcp"]
+# `data-beans plot`: QC and statistics figures (PNG, SVG, PDF) through
+# legume-plot, and the MCP `plot` tool.
+plot = ["dep:legume-plot", "dep:base64"]
 alg = ["aux"]
 sim = ["alg"]
 # Pulls in libhdf5 — opt-in so installs on hosts without HDF5 headers still
@@ -90,8 +96,15 @@ statrs = { version = "0.18", default-features = false }
 ratatui = { version = "0.30", default-features = false, features = ["crossterm"], optional = true }
 # Detachable sessions for the views; no pictures, so no graphics.
 legume-tui = { version = "0.3.2", optional = true, default-features = false, features = ["clap", "logs"] }
+# Its `tui` view (`data-beans mcp watch`) comes on with data-beans' `tui`.
+legume-mcp = { version = "0.1", optional = true }
+legume-plot = { version = "0.3.2", optional = true }
+base64 = { version = "0.22", optional = true }
 
 [dev-dependencies]
 # The crate's harness, driving the views through pseudo-terminals in
 # tests/detach_session.rs.
 legume-tui = { version = "0.3.2", default-features = false, features = ["testing"] }
+# tests/mcp_server.rs follows a server through the watch pane.
+legume-mcp = { version = "0.1", features = ["tui"] }
+ratatui = { version = "0.30", default-features = false, features = ["crossterm"] }

README.md

@@ -32,3 +32,128 @@ Here, we implement a wrapper for sparse matrix backends for swift data access by
         ├── indices (column indices)
         └── indptr (row pointers)
 ```
+
+## Agents (MCP)
+
+`data-beans mcp` serves data-beans to an AI agent as [Model Context Protocol](https://modelcontextprotocol.io) tools over stdio, for any MCP client: Claude Code, Claude Desktop, Cursor, VS Code, Gemini CLI, Codex CLI and others. The server talks only to the client that started it, over stdin and stdout; data-beans sends nothing anywhere itself. What leaves your machine is up to the agent's own client and model.
+
+### Connect a client
+
+The server is in the default build (the `mcp` feature; `cargo install data-beans` has it). Importing `.h5ad` and 10x `.h5` files needs `--features hdf5` too.
+
+The server keeps its records in a project folder: its working directory, or `LEGUME_MCP_PROJECT`. Set it for clients that start servers somewhere else (Claude Desktop, say).
+
+Most clients (Claude Desktop, Cursor, Gemini CLI) take an `mcpServers` entry:
+
+```json
+{
+  "mcpServers": {
+    "data-beans": {
+      "command": "data-beans",
+      "args": ["mcp"],
+      "env": { "LEGUME_MCP_PROJECT": "/path/to/project" }
+    }
+  }
+}
+```
+
+Claude Code, from the project folder:
+
+```sh
+claude mcp add data-beans -e LEGUME_MCP_PROJECT="$PWD" -- data-beans mcp
+```
+
+VS Code (`.vscode/mcp.json`):
+
+```json
+{
+  "servers": {
+    "data-beans": {
+      "type": "stdio",
+      "command": "data-beans",
+      "args": ["mcp"],
+      "env": { "LEGUME_MCP_PROJECT": "${workspaceFolder}" }
+    }
+  }
+}
+```
+
+Codex CLI (`~/.codex/config.toml`):
+
+```toml
+[mcp_servers.data-beans]
+command = "data-beans"
+args = ["mcp"]
+env = { LEGUME_MCP_PROJECT = "/path/to/project" }
+```
+
+### What the agent is told
+
+The server hands these instructions to the client when it connects. Clients that do not pass MCP instructions on to the model can paste them into an `AGENTS.md` or the system prompt:
+
+```text
+data-beans stores sparse single-cell matrices, features (genes, peaks) as rows and cells as columns, in .zarr.zip, .zarr or .h5 backends.
+
+Look before you derive:
+1. info gives a backend's size and metadata; list_contents shows what an outside .zarr or .h5 holds before you import it.
+2. names pages through row or column names, with a regex filter, to find the exact names to pass on.
+3. statistics summarizes each row or column (nnz, tot, mu, sig) with quantiles and the top entries by a chosen statistic; values reads the counts of a few named rows or columns.
+
+Quality control: qc runs the cell QC senna and pinto apply (a near-empty floor, then robust MAD bands on detected genes, total counts and the mito fraction). Call it without `out` first: it reports each metric's median, the band and how many cells fall outside, and writes nothing. Adjust n_mads or the caps until the drops are what the data warrant, then call it with `out` to write the kept cells as a new backend. plot shows the same settings as a figure (histograms with the band edges, counts against genes by verdict), so look at it before and after tuning.
+
+Derive new backends with import (mtx, zarr triplets, fragments, h5ad, 10x; say in `source` where the input came from), subset, subsample, split, convert and merge. Each writes to `out`, which must not exist yet: no tool replaces a file. Leave `out` out and the output goes in today's results folder, named after its input and the tool. Outputs are .zarr.zip archives unless `out` ends in .zarr (a directory) or .h5. Paths are resolved against the server's working directory; results give the paths written.
+
+A person may be watching (`data-beans mcp watch`): a "Note from the user" in a result is theirs to you; take it into account. A call they stopped fails with "stopped by the user": do not run it again unasked.
+```
+
+### Tools
+
+| Tool | What it does | Writes |
+|---|---|---|
+| `info` | a backend's size (rows, columns, non-zeros) and metadata | |
+| `list_contents` | the groups and arrays in a `.zarr`, `.zarr.zip` or `.h5`, before importing it | |
+| `names` | row or column names, paged, filtered by a case-insensitive regex | |
+| `statistics` | per-row or per-column nnz, total, mean and sd: quantiles and the top entries | a `.parquet` when `out` is given |
+| `values` | the counts of up to 100 named or indexed rows or columns | a table when `out` is given |
+| `qc` | senna's and pinto's cell QC: the near-empty floor, MAD bands on genes, counts and mito fraction, caps | the kept cells when `out` is given; a dry run otherwise |
+| `plot` | the QC figure, or a statistic's histogram, returned as an image | the figure |
+| `import` | a backend from `.mtx`, zarr triplets, fragments, `.h5ad` or 10x `.h5`, its `source` noted | a backend |
+| `subset` | some rows or columns: by `names`, a `pattern` (regex) or `indices` (ranges such as `0-19612`) | a backend |
+| `subsample` | cells and/or genes drawn at random | a backend |
+| `split` | cells split train/test, or into K folds; by region or group with `options` | the halves |
+| `convert` | zarr ↔ h5, `.zarr` ↔ `.zarr.zip` | a backend |
+| `merge` | backends side by side, batch-labelled; `common_rows` keeps the rows all share | a backend and its `.batch.gz` |
+| `notebook` | the agent's plan, observations and conclusions (from legume-mcp) | the day's `notebook.md` |
+
+No tool replaces a file: an output that is an input, or anything named after the output beside it, is refused, and `squeeze`, which rewrites its input, is not offered.
+
+### Records
+
+The server keeps the project's records the way [legume-mcp](https://crates.io/crates/legume-mcp) lays them out, after Noble's guide to organizing computational biology projects [1]:
+
+- outputs the agent does not name go in `results/<YYYY-MM-DD>/`, named after their input and the tool (`pbmc.imported.qc.subset.zarr.zip`);
+- every step that writes goes into the day's `results/<YYYY-MM-DD>/runall.sh`, with paths relative to the project and steps skipped when their outputs exist, so a copy of the project replays the session;
+- `results/INDEX.tsv` lists each output with a one-line summary;
+- `import` notes where its input came from (the agent's `source`: a URL, an accession, a version) in `data/README.md`;
+- the agent keeps `results/<YYYY-MM-DD>/notebook.md` through the `notebook` tool.
+
+`LEGUME_MCP_RECORDS=off` keeps no records.
+
+[1] W. S. Noble, "A quick guide to organizing computational biology projects", PLoS Comput Biol 5(7): e1000424 (2009). doi:[10.1371/journal.pcbi.1000424](https://doi.org/10.1371/journal.pcbi.1000424)
+
+### Watching
+
+```sh
+data-beans mcp watch
+```
+
+follows a running data-beans server in a detachable session: the agent's calls as they come, each one's log and result, the notebook. `s` stops the selected call, `n` leaves the agent a note (it arrives with its next result, and goes into the notebook), `o` opens a backend the call wrote in the statistics explorer, `F` follows again after you move. `--watch-only` offers no stop or note. With several servers running it asks which; `data-beans mcp watch <folder>` names one. `LEGUME_MCP_WATCH=off` keeps a server from being watched.
+
+### The same on the command line
+
+```sh
+data-beans qc data.zarr.zip --qc-mito-pattern '(?i)^MT-'            # summary only
+data-beans qc data.zarr.zip --qc-mito-pattern '(?i)^MT-' -o clean   # and the kept cells
+data-beans plot qc data.zarr.zip --qc-mito-pattern '(?i)^MT-' -o qc.png
+data-beans plot stat data.zarr.zip -s row --stat sum -o genes.png
+```

src/handlers/analysis.rs

@@ -12,7 +12,6 @@ use legume_numeric::matrix::common_io::*;
 use legume_numeric::matrix::membership::Membership;
 use legume_numeric::matrix::traits::RunningStatOps;
 use log::{info, warn};
-use regex::Regex;
 
 #[derive(ValueEnum, Clone, Debug, PartialEq)]
 #[clap(rename_all = "lowercase")]
@@ -263,14 +262,9 @@ fn rows_matching(cmd_args: &RunStatArgs, data: &SparseIoVec) -> anyhow::Result<O
     let Some(pattern) = &cmd_args.row_name_pattern else {
         return Ok(None);
     };
-    let re = Regex::new(&format!("(?i){}", pattern))
-        .map_err(|e| anyhow::anyhow!("invalid --row-name-pattern: {e}"))?;
     let row_names = data.row_names()?;
-    let selected: Vec<usize> = row_names
-        .iter()
-        .enumerate()
-        .filter_map(|(i, name)| re.is_match(name).then_some(i))
-        .collect();
+    let selected = crate::utilities::name_matching::names_matching(&row_names, pattern)
+        .map_err(|e| anyhow::anyhow!("--row-name-pattern: {e}"))?;
     info!(
         "Row pattern '{}' matched {}/{} rows",
         pattern,
@@ -285,13 +279,17 @@ fn rows_matching(cmd_args: &RunStatArgs, data: &SparseIoVec) -> anyhow::Result<O
 #[clap(rename_all = "lowercase")]
 pub enum HistStat {
     Nnz,
+    // `statistics` heads its columns tot, mu and sig.
+    #[value(alias = "tot")]
     Sum,
+    #[value(alias = "mu")]
     Mean,
+    #[value(alias = "sig")]
     Sd,
 }
 
 impl HistStat {
-    fn metric(&self) -> &'static str {
+    pub(crate) fn metric(&self) -> &'static str {
         match self {
             HistStat::Nnz => "nnz",
             HistStat::Sum => "sum",
@@ -302,7 +300,7 @@ impl HistStat {
 
     /// Pull the chosen statistic out of a running-stat accumulator (works for
     /// both the row and column variants via the shared `RunningStatOps` trait).
-    fn values<S: RunningStatOps<f32, Output = Vec<f32>>>(&self, s: &S) -> Vec<f32> {
+    pub(crate) fn values<S: RunningStatOps<f32, Output = Vec<f32>>>(&self, s: &S) -> Vec<f32> {
         match self {
             HistStat::Nnz => s.count_positives(),
             HistStat::Sum => s.sum(),