diff --git a/docs/adr/0028-agent-prompt-context-and-memory-model.md b/docs/adr/0028-agent-prompt-context-and-memory-model.md index 6eeee8e..8eefce4 100644 --- a/docs/adr/0028-agent-prompt-context-and-memory-model.md +++ b/docs/adr/0028-agent-prompt-context-and-memory-model.md @@ -1,6 +1,6 @@ # 0028. Agent prompt / context & conversation-memory model -- Status: Proposed +- Status: Accepted (Option A — bounded context-assembly policy — shipped; session store / RAG deferred) - Date: 2026-06-02 - Deciders: FUSE maintainers @@ -67,6 +67,14 @@ multi-turn requirements are concrete. ## More Information +- **Option A shipped**: `ai/agent` accepts optional `maxContextTokens` (an approximate token budget) + and `contextStrategy` (`drop-oldest` default | `summarize`). Before each completion the running + transcript is bounded by `trimContext` (`internal/packages/functions/ai/context.go`), which + preserves the leading system turns + the first user task and the most recent turns, dropping the + oldest middle turns; `summarize` replaces them with one LLM-generated summary turn (an extra, + opt-in model call — recorded as a `steps` entry). Token estimation is a coarse ~4-chars/token + heuristic, centralized so a real tokenizer can replace it. Absent budget = unchanged behavior. + **Deferred**: Option B (persisted cross-run session store) and Option C (vector/RAG memory). - Current assembly: `internal/packages/functions/ai/agent.go` (per-run message slice). - Related: [ADR-0007](0007-agent-reasoning-loop-and-tools-from-functions.md), [ADR-0019](0019-object-store-payload-externalization.md), diff --git a/docs/adr/README.md b/docs/adr/README.md index 4037fa4..836544d 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -54,7 +54,7 @@ copying [`template.md`](template.md). | 0025 | [Browser automation & web-scraping package](0025-browser-automation-and-web-scraping-package.md) | Proposed | 2026-06-01 | | 0026 | [Agent-as-orchestrator mode](0026-agent-as-orchestrator-mode.md) | Proposed | 2026-06-02 | | 0027 | [Async tool invocation via a sub-execution channel](0027-async-tool-invocation-sub-execution-channel.md) | Proposed | 2026-06-02 | -| 0028 | [Agent prompt/context & conversation-memory model](0028-agent-prompt-context-and-memory-model.md) | Proposed | 2026-06-02 | +| 0028 | [Agent prompt/context & conversation-memory model](0028-agent-prompt-context-and-memory-model.md) | Accepted | 2026-06-02 | | 0029 | [LLM cost & token-usage tracking and budgets](0029-llm-cost-and-usage-tracking-and-budgets.md) | Accepted | 2026-06-02 | | 0030 | [Structured/JSON output enforcement for ai nodes](0030-structured-output-enforcement.md) | Proposed | 2026-06-02 | | 0031 | [Settings, secrets & environments: a SecretStore seam](0031-settings-secrets-and-environments.md) | Accepted | 2026-06-02 | diff --git a/internal/packages/functions/ai/agent.go b/internal/packages/functions/ai/agent.go index 55f165d..ec78d86 100644 --- a/internal/packages/functions/ai/agent.go +++ b/internal/packages/functions/ai/agent.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "time" "github.com/open-source-cloud/fuse/internal/packages/transport" @@ -42,6 +43,8 @@ func AgentFunctionMetadata() workflow.FunctionMetadata { {Name: "temperature", Type: "float", Required: false, Description: "Sampling temperature; if omitted the provider default is used"}, {Name: "maxIterations", Type: "int", Required: false, Default: defaultMaxIterations, Description: "Maximum reasoning iterations (clamped to [1, 25])"}, {Name: "allowedTools", Type: "array", Required: false, Description: "Optional allowlist of full function ids the agent may use as tools; empty means all eligible tools"}, + {Name: "maxContextTokens", Type: "int", Required: false, Description: "Optional token budget for the running transcript (approximate); 0/absent disables trimming (ADR-0028)"}, + {Name: "contextStrategy", Type: "string", Required: false, Description: "When over maxContextTokens: 'drop-oldest' (default) or 'summarize' (an extra LLM call summarizes dropped turns)"}, }, Edges: workflow.InputEdgeMetadata{ Count: 0, @@ -81,10 +84,12 @@ func makeAgentFunction(providers llm.Registry, tools ToolRegistry, usage UsageRe model: input.GetStr("model"), temp: optionalTemperature(input), maxIters: clampIterations(input.GetInt("maxIterations")), - wfID: execInfo.WorkflowID, - execID: execInfo.ExecID, - environment: execInfo.Environment, - usage: usage, + wfID: execInfo.WorkflowID, + execID: execInfo.ExecID, + environment: execInfo.Environment, + usage: usage, + maxContextTokens: input.GetInt("maxContextTokens"), + contextStrategy: contextStrategyOrDefault(input.GetStr("contextStrategy")), } messages := make([]llm.Message, 0, 4) @@ -125,10 +130,12 @@ type agentExecutor struct { model string temp *float32 maxIters int - wfID workflow.ID - execID workflow.ExecID - environment string - usage UsageRecorder + wfID workflow.ID + execID workflow.ExecID + environment string + usage UsageRecorder + maxContextTokens int + contextStrategy string } // run drives the reasoning loop until a final answer, an error, or the iteration @@ -138,6 +145,12 @@ func (e *agentExecutor) run(ctx context.Context, messages []llm.Message) workflo steps := make([]map[string]any, 0) for i := 0; i < e.maxIters; i++ { + // Bound the growing transcript to the configured token budget (ADR-0028). + if trimmed, step := e.applyContextPolicy(ctx, messages); step != nil { + messages = trimmed + steps = append(steps, step) + } + resp, err := e.provider.Chat(ctx, llm.ChatRequest{ Model: e.model, Messages: messages, @@ -173,6 +186,52 @@ func (e *agentExecutor) run(ctx context.Context, messages []llm.Message) workflo return errorOutput("ai/agent: max iterations reached") } +// applyContextPolicy bounds the running transcript to maxContextTokens (ADR-0028). It returns the +// trimmed messages and a step record, or (nil, nil) when no trimming was needed. +func (e *agentExecutor) applyContextPolicy(ctx context.Context, messages []llm.Message) ([]llm.Message, map[string]any) { + if e.maxContextTokens <= 0 { + return nil, nil + } + kept, dropped := trimContext(messages, e.maxContextTokens) + if len(dropped) == 0 { + return nil, nil + } + step := map[string]any{"context": "trimmed", "strategy": e.contextStrategy, "droppedTurns": len(dropped)} + if e.contextStrategy == contextStrategySummarize { + if summary := e.summarizeDropped(ctx, dropped); summary != "" { + h := headLen(messages) + summaryTurn := llm.Message{Role: llm.RoleSystem, Content: "Summary of earlier turns: " + summary} + out := concatMessages(messages[:h], append([]llm.Message{summaryTurn}, kept[h:]...)) + step["summarized"] = true + return out, step + } + } + return kept, step +} + +// summarizeDropped asks the provider to summarize the dropped turns into one note. Returns "" on +// error, so the caller falls back to plain drop-oldest. +func (e *agentExecutor) summarizeDropped(ctx context.Context, dropped []llm.Message) string { + var b strings.Builder + for _, m := range dropped { + fmt.Fprintf(&b, "%s: %s\n", m.Role, m.Content) + } + resp, err := e.provider.Chat(ctx, llm.ChatRequest{ + Model: e.model, + Messages: []llm.Message{ + {Role: llm.RoleSystem, Content: "Summarize the following conversation turns concisely, preserving facts, tool results, and decisions."}, + {Role: llm.RoleUser, Content: b.String()}, + }, + }) + if err != nil { + log.Warn().Err(err).Msg("ai/agent context summarization failed; dropping oldest turns instead") + return "" + } + e.usage.RecordCall(AgentFunctionID, e.provider.Name(), e.model, "success") + e.usage.RecordUsage(AgentFunctionID, e.provider.Name(), e.model, resp.Usage) + return resp.Message.Content +} + // executeToolCall resolves, invokes, and records a single model-requested tool // call. It never aborts the run: unknown tools, bad arguments, invocation errors, // and tool errors are all fed back to the model as the tool's result so it can diff --git a/internal/packages/functions/ai/context.go b/internal/packages/functions/ai/context.go new file mode 100644 index 0000000..4fdbaee --- /dev/null +++ b/internal/packages/functions/ai/context.go @@ -0,0 +1,76 @@ +package ai + +import "github.com/open-source-cloud/fuse/pkg/llm" + +const ( + // contextStrategyDropOldest drops the oldest middle turns when over budget (deterministic). + contextStrategyDropOldest = "drop-oldest" + // contextStrategySummarize replaces dropped turns with a single LLM-generated summary turn + // (opt-in; introduces an extra model call and nondeterminism — ADR-0028). + contextStrategySummarize = "summarize" + // approxCharsPerToken is the heuristic used by estimateTokens. It is a coarse budget guard, + // not exact billing; centralized so a real tokenizer can replace it later. + approxCharsPerToken = 4 +) + +// contextStrategyOrDefault normalizes the configured strategy, defaulting to drop-oldest. +func contextStrategyOrDefault(s string) string { + if s == contextStrategySummarize { + return contextStrategySummarize + } + return contextStrategyDropOldest +} + +// estimateTokens approximates the token count of a message slice (~4 chars/token), counting both +// text content and tool-call arguments. +func estimateTokens(messages []llm.Message) int { + chars := 0 + for _, m := range messages { + chars += len(m.Content) + for _, tc := range m.ToolCalls { + chars += len(tc.Name) + len(tc.Arguments) + } + } + return chars / approxCharsPerToken +} + +// headLen returns the number of leading messages to always preserve: any leading system +// message(s) plus the first user message (the original task). +func headLen(messages []llm.Message) int { + h := 0 + for h < len(messages) && messages[h].Role == llm.RoleSystem { + h++ + } + if h < len(messages) && messages[h].Role == llm.RoleUser { + h++ + } + return h +} + +// trimContext bounds messages to budget tokens (ADR-0028). It preserves the head (leading system +// turns + first user task) and the most recent turns that fit, returning the oldest middle turns +// as dropped. budget<=0, already-fitting input, or a case where even the recent turns alone exceed +// the budget returns the input unchanged with no dropped turns. +func trimContext(messages []llm.Message, budget int) (kept, dropped []llm.Message) { + if budget <= 0 || estimateTokens(messages) <= budget { + return messages, nil + } + h := headLen(messages) + rest := messages[h:] + start := 0 + for start < len(rest) && estimateTokens(concatMessages(messages[:h], rest[start:])) > budget { + start++ + } + if start == 0 { + return messages, nil + } + return concatMessages(messages[:h], rest[start:]), rest[:start] +} + +// concatMessages returns a fresh slice of a followed by b. +func concatMessages(a, b []llm.Message) []llm.Message { + out := make([]llm.Message, 0, len(a)+len(b)) + out = append(out, a...) + out = append(out, b...) + return out +} diff --git a/internal/packages/functions/ai/context_test.go b/internal/packages/functions/ai/context_test.go new file mode 100644 index 0000000..02fe484 --- /dev/null +++ b/internal/packages/functions/ai/context_test.go @@ -0,0 +1,77 @@ +package ai + +import ( + "strings" + "testing" + + "github.com/open-source-cloud/fuse/pkg/llm" + "github.com/stretchr/testify/assert" +) + +// msg builds a message of n tokens (~n*4 chars) for a role. +func msg(role llm.Role, n int) llm.Message { + return llm.Message{Role: role, Content: strings.Repeat("x", n*approxCharsPerToken)} +} + +func TestContextStrategyOrDefault(t *testing.T) { + t.Parallel() + assert.Equal(t, contextStrategyDropOldest, contextStrategyOrDefault("")) + assert.Equal(t, contextStrategyDropOldest, contextStrategyOrDefault("nonsense")) + assert.Equal(t, contextStrategySummarize, contextStrategyOrDefault("summarize")) +} + +func TestTrimContext(t *testing.T) { + t.Parallel() + + // head = system(10) + user(10) = 20 tokens; then 4 middle turns of 10 each. + build := func() []llm.Message { + return []llm.Message{ + msg(llm.RoleSystem, 10), + msg(llm.RoleUser, 10), + msg(llm.RoleAssistant, 10), + msg(llm.RoleTool, 10), + msg(llm.RoleAssistant, 10), + msg(llm.RoleTool, 10), + } + } + + t.Run("budget disabled returns unchanged", func(t *testing.T) { + t.Parallel() + in := build() + kept, dropped := trimContext(in, 0) + assert.Len(t, kept, len(in)) + assert.Empty(t, dropped) + }) + + t.Run("under budget returns unchanged", func(t *testing.T) { + t.Parallel() + in := build() // 60 tokens + kept, dropped := trimContext(in, 1000) + assert.Len(t, kept, len(in)) + assert.Empty(t, dropped) + }) + + t.Run("over budget drops oldest middle, preserves head + recent", func(t *testing.T) { + t.Parallel() + in := build() // 60 tokens total; head=20 + // budget 40 -> head(20) + at most 2 recent turns (20) fit; drop the 2 oldest middle. + kept, dropped := trimContext(in, 40) + + // head preserved: first two are the original system + user task. + assert.Equal(t, llm.RoleSystem, kept[0].Role) + assert.Equal(t, llm.RoleUser, kept[1].Role) + // the two most recent turns are retained as the tail. + assert.Equal(t, in[len(in)-1], kept[len(kept)-1]) + assert.Equal(t, in[len(in)-2], kept[len(kept)-2]) + assert.Len(t, dropped, 2) + assert.LessOrEqual(t, estimateTokens(kept), 40) + }) + + t.Run("recent turns alone exceeding budget leaves input unchanged", func(t *testing.T) { + t.Parallel() + in := []llm.Message{msg(llm.RoleSystem, 10), msg(llm.RoleUser, 100)} + kept, dropped := trimContext(in, 5) + assert.Len(t, kept, len(in)) + assert.Empty(t, dropped) + }) +}