From 48767ff8914250a14f40e9cf36275c66a2cf45c9 Mon Sep 17 00:00:00 2001 From: Junjun Zhang Date: Thu, 17 Sep 2026 02:39:18 +0800 Subject: [PATCH] feat(agent): harness fingerprint for scaffolding-regression attribution Stamp the harness configuration (composed system prompt, write-integrity check registry, tool schema set) as a SHA-256 fingerprint into the debug log under the 'harness' category, logging on first observation and again only when any surface changes mid-session. Rationale (frontier research): - arXiv:2607.03691 'Don't Blame the LLM': across 35 releases of a coding agent with a FIXED model, resolve rate swung 23%-39% with no trend while token spend rose 70%+ - the swings tracked scaffolding changes. Core recommendation: 'report and control for the scaffolding version'. - Anthropic Claude Code postmortem (2026-04-23): weeks of 'it feels dumber' reports resolved to three harness changes; evals missed all. - arXiv:2607.06184 (TraceProbe): trajectory diffing for regression localization is only meaningful when the producing harness is recorded. ggcode previously had no way to attribute a debug export to the specific scaffolding that produced it: prompt composition, the ~191-check registry (e.g. the fc5c4aad critical-only trim), and MCP/tool registration could all change invisibly. The fingerprint closes that attribution gap so quality regressions can be bisected to a scaffolding change instead of being misattributed to the model. Wired via defer in maybeInjectDynamicSystemPrompt (fires on every agent Run), change-gated to avoid log spam. Zero behavior change otherwise. Co-Authored-By: ggcode --- internal/agent/agent.go | 1 + internal/agent/agent_prompt_inject.go | 6 + internal/agent/harness_fingerprint.go | 146 +++++++++++++++++++++ internal/agent/harness_fingerprint_test.go | 99 ++++++++++++++ 4 files changed, 252 insertions(+) create mode 100644 internal/agent/harness_fingerprint.go create mode 100644 internal/agent/harness_fingerprint_test.go diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 3f78445e8..0ea589c47 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -313,6 +313,7 @@ type Agent struct { systemPromptInjector func() string // returns extra system prompt text to inject (e.g. lanchat peer warnings) baseSystemPrompt string // the fully built static system prompt; used as reset base for dynamic injection lastInjectedSystemPrompt string // cache of last injected prompt to skip redundant updates + harnessFPLast string // last harness fingerprint sum; stamps scaffolding changes into the debug log (harness_fingerprint.go) onVerifyProgress func(text string) // called during async verification (status updates) onVerifyResult func(VerifyResult) // called when async verification completes onToolProgress func(toolID, toolName, output string) // called for streaming tool output (e.g. wait_command) diff --git a/internal/agent/agent_prompt_inject.go b/internal/agent/agent_prompt_inject.go index 13d7cab67..eccae3e05 100644 --- a/internal/agent/agent_prompt_inject.go +++ b/internal/agent/agent_prompt_inject.go @@ -33,6 +33,12 @@ func (a *Agent) maybeInjectDynamicSystemPrompt() { fn := a.systemPromptInjector a.mu.Unlock() + // Stamp the harness configuration (system prompt + check registry + tool + // set) into the debug ring. Deferred so it captures the prompt rebuilt by + // this function on every return path; logs only on change (first run or + // scaffolding mutation). See harness_fingerprint.go for rationale. + defer a.logHarnessFingerprint() + // Collect dynamic layers. var dynamicParts []string diff --git a/internal/agent/harness_fingerprint.go b/internal/agent/harness_fingerprint.go new file mode 100644 index 000000000..7a13fda93 --- /dev/null +++ b/internal/agent/harness_fingerprint.go @@ -0,0 +1,146 @@ +package agent + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strings" + "time" + + "github.com/topcheer/ggcode/internal/debug" + "github.com/topcheer/ggcode/internal/tool" +) + +// Harness Fingerprint: scaffolding-regression attribution for agent runs. +// +// Research basis (frontier agent literature, 2026): +// - arXiv:2607.03691 "Don't Blame the Large Language Model: How Scaffolding +// Evolution Shapes Coding Agent Quality" — across 35 releases of a coding +// agent with a FIXED underlying model, the resolve rate fluctuated between +// 23% and 39% with no upward trend while per-task token spend rose by 70%+ +// and completion time doubled. The swings tracked scaffolding changes, not +// model capability. The paper's core recommendation: maintain "report and +// control for the scaffolding version" alongside every run. +// - Anthropic's Claude Code postmortem (2026-04-23): six weeks of "it feels +// dumber" complaints resolved to three HARNESS changes (a reasoning-effort +// default, a prompt-caching bug, and a system-prompt edit measured at -3% +// quality); internal evals had missed all three. +// - arXiv:2607.06184 (TraceProbe): trajectory-level structural features, not +// outcome alone, localize regressions — but only if the harness that +// produced the trajectory is recorded, otherwise diffs are meaningless. +// +// ggcode's harness has three mutable surfaces that shape every run yet were +// previously invisible in trajectories and debug exports: +// +// 1. the composed system prompt (base + dynamic layers + ratchet rules), +// 2. the write-integrity check registry (allChecks, ~191 detectors edited +// across releases — e.g. the fc5c4aad critical-only trim), +// 3. the tool schema set (built-ins + MCP/plugins; can change mid-session). +// +// The fingerprint hashes all three and stamps each change into the debug log +// under the "harness" category (surfaced by debug log export), so a quality +// regression can be bisected to a specific scaffolding change instead of +// being misattributed to the model. + +// sha256Prefix returns the first 16 hex chars (64 bits) of the SHA-256 — +// collision probability is negligible for change attribution. +func sha256Prefix(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:])[:16] +} + +// hashNames hashes a list of identifiers in a stable (sorted) order. +func hashNames(names []string) (string, int) { + sorted := make([]string, len(names)) + copy(sorted, names) + sort.Strings(sorted) + return sha256Prefix(strings.Join(sorted, "\x00")), len(sorted) +} + +// integrityCheckNames lists the registered write-integrity check names. +// The registry is initialized lazily (check_registry.go); before the first +// registration the list is simply empty — and that transition is itself a +// harness change worth stamping into the log. +func integrityCheckNames() []string { + names := make([]string, 0, len(allChecks)) + for _, c := range allChecks { + names = append(names, c.Name) + } + return names +} + +// harnessToolNames lists the registered tool names from an agent's registry. +func harnessToolNames(reg *tool.Registry) []string { + if reg == nil { + return nil + } + names := make([]string, 0, 32) + for _, t := range reg.List() { + names = append(names, t.Name()) + } + return names +} + +// HarnessFingerprint is a stable digest of the harness configuration that +// produced a given agent run. Two runs with identical fingerprints executed +// under identical scaffolding; a difference localizes exactly which surface +// moved (prompt vs. checks vs. tools). +type HarnessFingerprint struct { + SystemPromptSHA string `json:"system_prompt_sha"` + SystemPromptLen int `json:"system_prompt_len"` + ChecksSHA string `json:"checks_sha"` + ChecksCount int `json:"checks_count"` + ToolsSHA string `json:"tools_sha"` + ToolsCount int `json:"tools_count"` + ComputedAt time.Time `json:"computed_at"` +} + +// Sum returns a compact stable identity for the whole fingerprint. +func (fp HarnessFingerprint) Sum() string { + return fmt.Sprintf("sp=%s/%d checks=%s/%d tools=%s/%d", + fp.SystemPromptSHA, fp.SystemPromptLen, + fp.ChecksSHA, fp.ChecksCount, + fp.ToolsSHA, fp.ToolsCount) +} + +// ComputeHarnessFingerprint snapshots the current harness configuration. +func (a *Agent) ComputeHarnessFingerprint() HarnessFingerprint { + a.mu.RLock() + prompt := a.baseSystemPrompt + reg := a.tools + a.mu.RUnlock() + + checksSHA, checksN := hashNames(integrityCheckNames()) + toolsSHA, toolsN := hashNames(harnessToolNames(reg)) + return HarnessFingerprint{ + SystemPromptSHA: sha256Prefix(prompt), + SystemPromptLen: len(prompt), + ChecksSHA: checksSHA, + ChecksCount: checksN, + ToolsSHA: toolsSHA, + ToolsCount: toolsN, + ComputedAt: time.Now(), + } +} + +// logHarnessFingerprint stamps harness changes into the debug ring. It logs +// once on first observation and again ONLY when a component changes, so a +// session with mid-flight tool registration or registry reconciliation +// produces an auditable change history instead of log spam. +func (a *Agent) logHarnessFingerprint() { + sum := a.ComputeHarnessFingerprint().Sum() + a.mu.Lock() + prev := a.harnessFPLast + a.harnessFPLast = sum + a.mu.Unlock() + + switch { + case prev == sum: + // unchanged — no log + case prev == "": + debug.Log("harness", "harness fingerprint: %s", sum) + default: + debug.Log("harness", "harness fingerprint CHANGED: %s (was %s)", sum, prev) + } +} diff --git a/internal/agent/harness_fingerprint_test.go b/internal/agent/harness_fingerprint_test.go new file mode 100644 index 000000000..b761ae44e --- /dev/null +++ b/internal/agent/harness_fingerprint_test.go @@ -0,0 +1,99 @@ +package agent + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/topcheer/ggcode/internal/tool" +) + +// stubToolForHarness is a minimal Tool implementation for fingerprint tests. +type stubToolForHarness struct{} + +func (stubToolForHarness) Name() string { return "test_tool" } +func (stubToolForHarness) Description() string { return "stub" } +func (stubToolForHarness) Parameters() json.RawMessage { + return json.RawMessage(`{"type":"object"}`) +} +func (stubToolForHarness) Execute(ctx context.Context, input json.RawMessage) (tool.Result, error) { + return tool.Result{}, nil +} + +// TestHarnessFingerprint_StableAndSensitive verifies the fingerprint is +// stable across repeated computation on an unchanged agent and sensitive to +// the harness surfaces it covers. +func TestHarnessFingerprint_StableAndSensitive(t *testing.T) { + a := &Agent{} + a.baseSystemPrompt = "base prompt v1" + + fp1 := a.ComputeHarnessFingerprint() + fp2 := a.ComputeHarnessFingerprint() + if fp1.Sum() != fp2.Sum() { + t.Fatalf("fingerprint unstable for unchanged harness: %s vs %s", fp1.Sum(), fp2.Sum()) + } + if !strings.Contains(fp1.Sum(), "sp=") || !strings.Contains(fp1.Sum(), "checks=") || !strings.Contains(fp1.Sum(), "tools=") { + t.Fatalf("Sum() missing components: %s", fp1.Sum()) + } + + // Surface 1: system prompt change must move the fingerprint. + a.baseSystemPrompt = "base prompt v2" + if a.ComputeHarnessFingerprint().Sum() == fp1.Sum() { + t.Fatal("fingerprint did not change after system prompt edit") + } + + // Surface 3: tool registration must move the fingerprint. + a.tools = tool.NewRegistry() + if err := a.tools.Register(stubToolForHarness{}); err != nil { + t.Fatalf("Register failed: %v", err) + } + fpTools := a.ComputeHarnessFingerprint() + if fpTools.Sum() == fp1.Sum() { + t.Fatal("fingerprint did not change after tool registration") + } + if fpTools.ToolsCount != 1 { + t.Fatalf("ToolsCount = %d, want 1", fpTools.ToolsCount) + } + + // nil registry must be tolerated (tools surface simply empty). + harnessToolNames(nil) +} + +// TestHashNames_OrderInsensitive verifies the sorted-hash contract: the same +// set of names in different orders yields the same SHA and count. +func TestHashNames_OrderInsensitive(t *testing.T) { + h1, n1 := hashNames([]string{"b", "a", "c"}) + h2, n2 := hashNames([]string{"c", "a", "b"}) + if h1 != h2 || n1 != n2 { + t.Fatalf("hashNames not order-insensitive: (%s,%d) vs (%s,%d)", h1, n1, h2, n2) + } + h3, n3 := hashNames([]string{"b", "a", "c", "d"}) + if h3 == h1 || n3 == n1 { + t.Fatal("hashNames failed to distinguish different sets") + } +} + +// TestLogHarnessFingerprint_ChangeDetection verifies the once-on-first and +// once-on-change logging contract at the state level. +func TestLogHarnessFingerprint_ChangeDetection(t *testing.T) { + a := &Agent{} + a.baseSystemPrompt = "p1" + + a.logHarnessFingerprint() + first := a.harnessFPLast + if first == "" { + t.Fatal("harnessFPLast not populated after first log") + } + + a.logHarnessFingerprint() // unchanged — must be a no-op + if a.harnessFPLast != first { + t.Fatal("unchanged harness altered fingerprint state") + } + + a.baseSystemPrompt = "p2" + a.logHarnessFingerprint() + if a.harnessFPLast == first { + t.Fatal("changed harness did not update fingerprint state") + } +}