From e77de87f8e9270992e565268464ee54def2f50ca Mon Sep 17 00:00:00 2001 From: Kyle Galloway Date: Fri, 31 Jul 2026 08:21:35 -0400 Subject: [PATCH] feat(search): hits carry file context, windowed reads, read_many MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three ergonomics fixes for the case memory is actually heading toward — more files, longer notes: - search hits include the containing file's description (H1 + first prose line, derived not authored), so a line lifted from a 30KB note is attributable without a second call - read_memory takes from_line/max_lines and reports total_lines + partial, so long notes can be read in windows - read_many reads a glob in one call with a per-file cap, replacing N round trips for 'show me all the notes' Adapted from ideas in tobi/qmd without the dependency: it loads its own in-process GGUF models, which would put a second model runtime and a second embedding universe next to the fleet's existing one. No index is added here — the substrate stays markdown in git and search stays grep. Co-Authored-By: Claude Fable 5 --- docs/design.md | 27 ++++++++ internal/mcpserver/server.go | 49 ++++++++++++-- internal/store/store.go | 128 ++++++++++++++++++++++++++++++++++- internal/store/store_test.go | 128 +++++++++++++++++++++++++++++++++++ 4 files changed, 325 insertions(+), 7 deletions(-) diff --git a/docs/design.md b/docs/design.md index 9fc45fd..70abdc0 100644 --- a/docs/design.md +++ b/docs/design.md @@ -43,3 +43,30 @@ tools serve deep memory reads and writes during the conversation. Tools alone fail because a model cannot search for what it doesn't know it forgot; injection alone fails because nothing new would ever be recorded. + + +## Search ergonomics (2026-07-31) + +Three refinements, adapted from ideas in tobi/qmd (a local markdown +search engine) without taking the dependency — it runs its own +in-process GGUF models, which would mean a second model runtime and a +second embedding universe beside the fleet's existing one. + +- **Hits carry their file's description.** A matching line lifted out of + a long note is ambiguous alone; every `search_memory` hit now includes + the containing file's H1 plus first prose line. Derived, never + authored — no frontmatter to maintain, no way for it to drift. +- **Reads can be windowed.** `read_memory` takes `from_line`/`max_lines` + and reports `total_lines` plus `partial`, so a long note can be read + in pieces and the caller can tell whether it has the whole thing. +- **`read_many` reads a glob in one call** with a per-file line cap. + "Show me all the notes" was N round trips; now it is one, and a wide + pattern stays affordable. + +None of this adds an index. The substrate is still markdown in git, and +search is still grep — appropriate while memory holds only the user's +own writing. The trigger for revisiting: searches that miss content the +user knows is there (paraphrase misses), or a memory directory growing +past a megabyte or two. The fleet-native answer then is an optional +index built against the *same* embedding service the knowledge base +uses, not a second one. diff --git a/internal/mcpserver/server.go b/internal/mcpserver/server.go index 7d378c3..2cf036a 100644 --- a/internal/mcpserver/server.go +++ b/internal/mcpserver/server.go @@ -20,6 +20,7 @@ import ( "fmt" "log/slog" "net/http" + "strings" "github.com/modelcontextprotocol/go-sdk/mcp" @@ -32,6 +33,8 @@ type memoryStore interface { CreateExpert(expert, description string) error ListFiles(expert string) ([]string, error) ReadFile(expert, file string) (string, error) + ReadFileLines(expert, file string, from, count int) (string, int, error) + ReadFiles(expert, pattern string, maxLinesPerFile int) ([]store.MultiFile, error) Search(expert, query string) ([]store.SearchHit, error) RecordDecision(expert, decision, why string) error RecordNote(expert, topic, content string) error @@ -71,11 +74,20 @@ func Handler(deps Deps) http.Handler { "which file matters — they are small.", }, deps.readMemory) + mcp.AddTool(server, &mcp.Tool{ + Name: "read_many", + Description: "Read every memory file matching a glob in one call — 'notes/*' for all " + + "topic notes, '*.md' for the core files. Each file comes back with its " + + "self-description and is capped at max_lines_per_file, so a wide pattern stays " + + "affordable. Prefer this over repeated read_memory calls.", + }, deps.readMany) + mcp.AddTool(server, &mcp.Tool{ Name: "search_memory", Description: "Search an expert's memory files for lines matching any query term. " + "Use for 'did we already decide/discuss X?' before answering questions about the " + - "user's plans or history.", + "user's plans or history. Each hit carries the containing file's description, " + + "so you can tell a decision from a stray mention without a second call.", }, deps.searchMemory) mcp.AddTool(server, &mcp.Tool{ @@ -180,20 +192,27 @@ func (d Deps) listExperts(ctx context.Context, _ *mcp.CallToolRequest, _ struct{ } type readInput struct { - Expert string `json:"expert" jsonschema:"expert slug from list_experts"` - File string `json:"file" jsonschema:"memory file: profile.md, goals.md, decisions.md, log.md, or notes/.md"` + Expert string `json:"expert" jsonschema:"expert slug from list_experts"` + File string `json:"file" jsonschema:"memory file: profile.md, goals.md, decisions.md, log.md, or notes/.md"` + FromLine int `json:"from_line,omitempty" jsonschema:"1-indexed first line to return; omit for the whole file"` + MaxLines int `json:"max_lines,omitempty" jsonschema:"maximum lines to return; omit for the whole file"` } type readOutput struct { - Content string `json:"content"` + Content string `json:"content"` + TotalLines int `json:"total_lines"` + // Partial says the window returned is less than the whole file, so a + // caller knows to ask for more rather than assume it has everything. + Partial bool `json:"partial,omitempty"` } func (d Deps) readMemory(ctx context.Context, _ *mcp.CallToolRequest, in readInput) (*mcp.CallToolResult, readOutput, error) { - content, err := d.Store.ReadFile(in.Expert, in.File) + content, total, err := d.Store.ReadFileLines(in.Expert, in.File, in.FromLine, in.MaxLines) if err != nil { return nil, readOutput{}, err } - return nil, readOutput{Content: content}, nil + returned := strings.Count(content, "\n") + 1 + return nil, readOutput{Content: content, TotalLines: total, Partial: returned < total}, nil } type searchInput struct { @@ -213,6 +232,24 @@ func (d Deps) searchMemory(ctx context.Context, _ *mcp.CallToolRequest, in searc return nil, searchOutput{Hits: hits}, nil } +type readManyInput struct { + Expert string `json:"expert" jsonschema:"expert slug"` + Pattern string `json:"pattern" jsonschema:"glob over memory-relative paths, e.g. notes/* or *.md"` + MaxLinesPerFile int `json:"max_lines_per_file,omitempty" jsonschema:"per-file line cap, default 200"` +} + +type readManyOutput struct { + Files []store.MultiFile `json:"files"` +} + +func (d Deps) readMany(ctx context.Context, _ *mcp.CallToolRequest, in readManyInput) (*mcp.CallToolResult, readManyOutput, error) { + files, err := d.Store.ReadFiles(in.Expert, in.Pattern, in.MaxLinesPerFile) + if err != nil { + return nil, readManyOutput{}, err + } + return nil, readManyOutput{Files: files}, nil +} + type decisionInput struct { Expert string `json:"expert" jsonschema:"expert slug"` Decision string `json:"decision" jsonschema:"the decision, one clear sentence first"` diff --git a/internal/store/store.go b/internal/store/store.go index cf47f6d..92556a2 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -273,6 +273,13 @@ type SearchHit struct { File string `json:"file"` Line int `json:"line"` Text string `json:"text"` + // About is the containing file's self-description: its H1 plus first + // prose line. A matching line lifted out of a 30KB note is ambiguous + // on its own — "biomass, 4-6 week lead" reads very differently under + // "Forestry plan" than under "Equipment inventory" — so every hit + // carries what it came from. Derived, never authored: no frontmatter + // to maintain and no way for it to drift from the file. + About string `json:"about,omitempty"` } // Search scans an expert's memory files for lines containing ANY of the @@ -294,11 +301,12 @@ func (s *Store) Search(expert, query string) ([]SearchHit, error) { if err != nil { continue } + about := describe(content) for i, line := range strings.Split(content, "\n") { lower := strings.ToLower(line) for _, t := range terms { if strings.Contains(lower, t) { - hits = append(hits, SearchHit{File: f, Line: i + 1, Text: strings.TrimSpace(line)}) + hits = append(hits, SearchHit{File: f, Line: i + 1, Text: strings.TrimSpace(line), About: about}) break } } @@ -424,6 +432,124 @@ func (s *Store) appendFile(expert, file, entry, msg string) error { return nil } +// describe derives a file's one-line self-description: its H1 title and +// the first prose line under it. Blank when a file has neither. +func describe(content string) string { + var title, first string + for _, line := range strings.Split(content, "\n") { + t := strings.TrimSpace(line) + if t == "" { + continue + } + if strings.HasPrefix(t, "# ") && title == "" { + title = strings.TrimSpace(t[2:]) + continue + } + if strings.HasPrefix(t, "#") || strings.HasPrefix(t, ">") { + continue // deeper headings and quotes are not a description + } + first = t + break + } + switch { + case title != "" && first != "": + return capRunes(title+" — "+first, 200) + case title != "": + return title + default: + return capRunes(first, 200) + } +} + +// sliceLines returns a 1-indexed line window of content. A zero from +// means the start; a zero or negative count means to the end. Slicing +// exists because a memory file can be long enough that reading it whole +// is the expensive thing an agent does in a turn. +func sliceLines(content string, from, count int) string { + lines := strings.Split(content, "\n") + if from > len(lines) { + return "" + } + if from > 1 { + lines = lines[from-1:] + } + if count > 0 && count < len(lines) { + lines = lines[:count] + } + return strings.Join(lines, "\n") +} + +// ReadFileLines reads a memory file, optionally windowed to a line +// range, and reports the file's total line count so a caller can tell +// whether it received the whole thing. +func (s *Store) ReadFileLines(expert, file string, from, count int) (text string, totalLines int, err error) { + content, err := s.ReadFile(expert, file) + if err != nil { + return "", 0, err + } + total := strings.Count(content, "\n") + 1 + if from <= 0 && count <= 0 { + return content, total, nil + } + return sliceLines(content, from, count), total, nil +} + +// MultiFile is one file returned by ReadFiles. +type MultiFile struct { + File string `json:"file"` + About string `json:"about,omitempty"` + Text string `json:"text"` + TotalLines int `json:"total_lines"` + Truncated bool `json:"truncated,omitempty"` +} + +// ReadFiles reads every memory file of an expert whose path matches a +// glob (filepath.Match semantics against the memory-relative path, e.g. +// "notes/*"), capping each file's returned lines. One call for "show me +// all the notes" beats N round trips, and the per-file cap keeps a wide +// pattern from blowing the caller's context. +func (s *Store) ReadFiles(expert, pattern string, maxLinesPerFile int) ([]MultiFile, error) { + files, err := s.ListFiles(expert) + if err != nil { + return nil, err + } + if strings.TrimSpace(pattern) == "" { + return nil, fmt.Errorf("pattern is required (e.g. \"notes/*\" or \"*.md\")") + } + if maxLinesPerFile <= 0 { + maxLinesPerFile = 200 + } + var out []MultiFile + for _, f := range files { + ok, err := filepath.Match(pattern, f) + if err != nil { + return nil, fmt.Errorf("bad pattern %q: %w", pattern, err) + } + if !ok { + continue + } + content, err := s.ReadFile(expert, f) + if err != nil { + continue + } + total := strings.Count(content, "\n") + 1 + text := content + truncated := false + if total > maxLinesPerFile { + text = sliceLines(content, 1, maxLinesPerFile) + truncated = true + } + out = append(out, MultiFile{ + File: f, About: describe(content), Text: text, + TotalLines: total, Truncated: truncated, + }) + } + if len(out) == 0 { + return nil, fmt.Errorf("no memory files match %q", pattern) + } + return out, nil +} + // session is the ephemeral scratch state, persisted outside git. type session struct { Context string `json:"context"` diff --git a/internal/store/store_test.go b/internal/store/store_test.go index cf4a934..c2c0940 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -2,6 +2,7 @@ package store import ( "errors" + "fmt" "os" "os/exec" "path/filepath" @@ -249,3 +250,130 @@ func TestMain(m *testing.M) { } os.Exit(m.Run()) } + +func TestDescribe(t *testing.T) { + cases := []struct { + name, content, want string + }{ + { + name: "title and first prose line", + content: "# Forestry plan\n\nScenario C deep dive: managed Acadian forest.\n\n## Details\n", + want: "Forestry plan — Scenario C deep dive: managed Acadian forest.", + }, + {"title only", "# Goals\n", "Goals"}, + {"prose only", "just a line of text\n", "just a line of text"}, + {"skips deeper headings and quotes", "# T\n\n> a quote\n\n## Sub\n\nreal line\n", "T — real line"}, + {"empty", "\n\n", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := describe(tc.content); got != tc.want { + t.Errorf("describe() = %q, want %q", got, tc.want) + } + }) + } +} + +func TestSearchHitsCarryFileDescription(t *testing.T) { + s := newTestStore(t) + if err := s.CreateExpert("x4", "d"); err != nil { + t.Fatalf("CreateExpert: %v", err) + } + if err := s.RecordNote("x4", "boarding", "Use three-star marines above 40% hull."); err != nil { + t.Fatalf("RecordNote: %v", err) + } + hits, err := s.Search("x4", "marines") + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(hits) != 1 { + t.Fatalf("hits = %d, want 1", len(hits)) + } + // A note created by RecordNote is "# " then the content, so the + // hit should be attributable without opening the file. + if !strings.Contains(hits[0].About, "boarding") { + t.Errorf("hit About = %q, want it to name the note", hits[0].About) + } +} + +func TestReadFileLines(t *testing.T) { + s := newTestStore(t) + if err := s.CreateExpert("x4", "d"); err != nil { + t.Fatalf("CreateExpert: %v", err) + } + body := "# Long\n\n" + for i := 1; i <= 50; i++ { + body += fmt.Sprintf("line %d\n", i) + } + if err := s.RecordNote("x4", "long", body); err != nil { + t.Fatalf("RecordNote: %v", err) + } + + whole, total, err := s.ReadFileLines("x4", "notes/long.md", 0, 0) + if err != nil { + t.Fatalf("ReadFileLines whole: %v", err) + } + if !strings.Contains(whole, "line 50") || total < 50 { + t.Fatalf("whole read wrong: total=%d", total) + } + + window, total2, err := s.ReadFileLines("x4", "notes/long.md", 5, 3) + if err != nil { + t.Fatalf("ReadFileLines window: %v", err) + } + if got := strings.Count(window, "\n") + 1; got != 3 { + t.Errorf("window lines = %d, want 3", got) + } + if total2 != total { + t.Errorf("total changed with window: %d vs %d", total2, total) + } + // Past the end is empty, not an error — a caller probing the tail of a + // file it has not measured should not get a failure. + if past, _, err := s.ReadFileLines("x4", "notes/long.md", 9999, 10); err != nil || past != "" { + t.Errorf("past-end read = %q, %v; want empty, nil", past, err) + } +} + +func TestReadFiles(t *testing.T) { + s := newTestStore(t) + if err := s.CreateExpert("x4", "d"); err != nil { + t.Fatalf("CreateExpert: %v", err) + } + for _, topic := range []string{"boarding", "mining"} { + if err := s.RecordNote("x4", topic, "content for "+topic+"\nmore\nlines\nhere\n"); err != nil { + t.Fatalf("RecordNote %s: %v", topic, err) + } + } + + files, err := s.ReadFiles("x4", "notes/*", 0) + if err != nil { + t.Fatalf("ReadFiles: %v", err) + } + if len(files) != 2 { + t.Fatalf("files = %d, want 2", len(files)) + } + for _, f := range files { + if f.About == "" || f.Text == "" || f.TotalLines == 0 { + t.Errorf("incomplete MultiFile: %+v", f) + } + } + + // The per-file cap must mark what it cut. + capped, err := s.ReadFiles("x4", "notes/boarding.md", 2) + if err != nil { + t.Fatalf("ReadFiles capped: %v", err) + } + if len(capped) != 1 || !capped[0].Truncated { + t.Fatalf("expected one truncated file, got %+v", capped) + } + if got := strings.Count(capped[0].Text, "\n") + 1; got != 2 { + t.Errorf("capped lines = %d, want 2", got) + } + + if _, err := s.ReadFiles("x4", "nothing/*", 0); err == nil { + t.Error("non-matching pattern did not error") + } + if _, err := s.ReadFiles("x4", "", 0); err == nil { + t.Error("empty pattern did not error") + } +}